00000020:00200000:1.0:1789760711.794018:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760711.794028:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760711.794061:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17301504, count: 4096 00000080:00200000:1.0:1789760711.794067:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17301504, bytes: 4096 00000080:00200000:1.0:1789760711.794079:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760711.794088:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760711.794090:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17301504, 17305600) 0 0 00000080:00200000:1.0:1789760711.794095:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760711.794110:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4224, 4224] 00020000:00200000:1.0:1789760711.794117:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17301504 chunk: [17301504, 17305600) 17305600, 4096 00020000:00200000:1.0:1789760711.794122:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760711.794139:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4224, 4224] 00000008:00010000:1.0:1789760711.794154:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760711.794161:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760711.794172:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17301504, 17305600) append 0 append lockpos 0 00000080:00200000:1.0:1789760711.794186:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17301504=0x1080000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760711.794252:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760711.794255:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17305600 00000020:00010000:1.0:1789760711.794264:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760711.794267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760711.794270:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760711.794274:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760711.794277:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760711.794284:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760711.794290:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760711.794305:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760711.794434:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 17301504<>17305600 00000080:00200000:1.0:1789760711.866613:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4224, 4224] 00000080:00200000:1.0:1789760711.866629:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760711.866636:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760711.866661:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17305600 i_blocks 34823 00000020:00200000:1.0:1789760711.866666:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760711.866677:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760711.866713:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17305600, count: 4096 00000080:00200000:1.0:1789760711.866721:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17305600, bytes: 4096 00000080:00200000:1.0:1789760711.866729:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760711.866737:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760711.866739:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17305600, 17309696) 0 0 00000080:00200000:1.0:1789760711.866744:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760711.866761:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4225, 4225] 00020000:00200000:1.0:1789760711.866769:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17305600 chunk: [17305600, 17309696) 17309696, 4096 00020000:00200000:1.0:1789760711.866774:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760711.866794:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4225, 4225] 00000008:00010000:1.0:1789760711.866811:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760711.866818:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760711.866830:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17305600, 17309696) append 0 append lockpos 0 00000080:00200000:1.0:1789760711.866845:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17305600=0x1081000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760711.866908:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760711.866911:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17309696 00000020:00010000:1.0:1789760711.866920:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760711.866924:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760711.866927:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760711.866943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760711.866947:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760711.866955:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760711.867160:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760711.867182:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760711.867306:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 17305600<>17309696 00000080:00200000:1.0:1789760711.965975:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4225, 4225] 00000080:00200000:1.0:1789760711.965992:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760711.965999:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760711.966015:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17309696 i_blocks 34823 00000020:00200000:1.0:1789760711.966019:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760711.966030:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760711.966067:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17309696, count: 4096 00000080:00200000:1.0:1789760711.966076:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17309696, bytes: 4096 00000080:00200000:1.0:1789760711.966085:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760711.966094:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760711.966097:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17309696, 17313792) 0 0 00000080:00200000:1.0:1789760711.966101:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760711.966120:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4226, 4226] 00020000:00200000:1.0:1789760711.966129:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17309696 chunk: [17309696, 17313792) 17313792, 4096 00020000:00200000:1.0:1789760711.966133:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760711.966154:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4226, 4226] 00000008:00010000:1.0:1789760711.966173:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760711.966179:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760711.966189:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17309696, 17313792) append 0 append lockpos 0 00000080:00200000:1.0:1789760711.966203:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17309696=0x1082000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760711.966276:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760711.966279:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17313792 00000020:00010000:1.0:1789760711.966289:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760711.966293:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760711.966296:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760711.966300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760711.966303:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760711.966312:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760711.966319:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760711.966338:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760711.966485:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 17309696<>17313792 00000080:00200000:1.0:1789760712.033778:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4226, 4226] 00000080:00200000:1.0:1789760712.033796:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.033803:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.033818:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17313792 i_blocks 34823 00000020:00200000:1.0:1789760712.033822:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.033833:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.033865:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17313792, count: 4096 00000080:00200000:1.0:1789760712.033873:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17313792, bytes: 4096 00000080:00200000:1.0:1789760712.033882:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.033892:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.033894:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17313792, 17317888) 0 0 00000080:00200000:1.0:1789760712.033899:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.033917:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4227, 4227] 00020000:00200000:1.0:1789760712.033925:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17313792 chunk: [17313792, 17317888) 17317888, 4096 00020000:00200000:1.0:1789760712.033930:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.033949:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4227, 4227] 00000008:00010000:1.0:1789760712.033980:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.033986:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.033998:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17313792, 17317888) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.034015:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17313792=0x1083000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.034067:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.034069:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17317888 00000020:00010000:1.0:1789760712.034078:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.034081:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.034084:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.034087:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.034091:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.034098:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.034105:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.131157:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4227, 4227] 00000080:00200000:1.0:1789760712.131175:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.131181:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.131198:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17317888 i_blocks 34823 00000020:00200000:1.0:1789760712.131201:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.131222:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.131266:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17317888, count: 4096 00000080:00200000:1.0:1789760712.131274:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17317888, bytes: 4096 00000080:00200000:1.0:1789760712.131285:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.131294:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.131297:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17317888, 17321984) 0 0 00000080:00200000:1.0:1789760712.131301:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.131319:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4228, 4228] 00020000:00200000:1.0:1789760712.131326:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17317888 chunk: [17317888, 17321984) 17321984, 4096 00020000:00200000:1.0:1789760712.131331:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.131352:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4228, 4228] 00000008:00010000:1.0:1789760712.131371:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.131412:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.131425:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17317888, 17321984) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.131442:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17317888=0x1084000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.131503:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.131506:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17321984 00000020:00010000:1.0:1789760712.131515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.131518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.131521:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.131524:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.131528:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.131535:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.131542:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.131560:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760712.131799:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 17317888<>17321984 00000080:00200000:1.0:1789760712.188572:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4228, 4228] 00000080:00200000:1.0:1789760712.188601:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.188608:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.188625:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17321984 i_blocks 34823 00000020:00200000:1.0:1789760712.188629:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.188640:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.188791:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17321984, count: 4096 00000080:00200000:1.0:1789760712.188801:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17321984, bytes: 4096 00000080:00200000:1.0:1789760712.188812:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.188822:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.188825:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17321984, 17326080) 0 0 00000080:00200000:1.0:1789760712.188829:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.188846:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4229, 4229] 00020000:00200000:1.0:1789760712.188854:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17321984 chunk: [17321984, 17326080) 17326080, 4096 00020000:00200000:1.0:1789760712.188859:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.188882:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4229, 4229] 00000008:00010000:1.0:1789760712.188902:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.188908:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.188919:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17321984, 17326080) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.188934:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17321984=0x1085000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.191309:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.191321:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17326080 00000020:00010000:1.0:1789760712.191335:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.191340:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.191345:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.191349:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.191352:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.191364:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.191398:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.191424:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760712.191550:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 17321984<>17326080 00000080:00200000:1.0:1789760712.238047:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4229, 4229] 00000080:00200000:1.0:1789760712.238064:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.238070:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.238086:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17326080 i_blocks 34823 00000020:00200000:1.0:1789760712.238090:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.238101:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.238138:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17326080, count: 4096 00000080:00200000:1.0:1789760712.238146:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17326080, bytes: 4096 00000080:00200000:1.0:1789760712.238154:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.238162:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.238164:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17326080, 17330176) 0 0 00000080:00200000:1.0:1789760712.238169:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.238185:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4230, 4230] 00020000:00200000:1.0:1789760712.238194:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17326080 chunk: [17326080, 17330176) 17330176, 4096 00020000:00200000:1.0:1789760712.238198:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.238229:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4230, 4230] 00000008:00010000:1.0:1789760712.238257:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.238263:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.238275:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17326080, 17330176) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.238289:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17326080=0x1086000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.238344:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.238347:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17330176 00000020:00010000:1.0:1789760712.238355:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.238359:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.238362:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.238365:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.238368:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.238415:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.238422:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.238437:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760712.238532:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 17326080<>17330176 00000080:00200000:1.0:1789760712.292685:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4230, 4230] 00000080:00200000:1.0:1789760712.292705:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.292713:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.292730:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17330176 i_blocks 34823 00000020:00200000:1.0:1789760712.292734:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.292746:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.292781:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17330176, count: 4096 00000080:00200000:1.0:1789760712.292790:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17330176, bytes: 4096 00000080:00200000:1.0:1789760712.292800:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.292809:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.292812:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17330176, 17334272) 0 0 00000080:00200000:1.0:1789760712.292817:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.292836:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4231, 4231] 00020000:00200000:1.0:1789760712.292845:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17330176 chunk: [17330176, 17334272) 17334272, 4096 00020000:00200000:1.0:1789760712.292850:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.292870:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4231, 4231] 00000008:00010000:1.0:1789760712.292890:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.292897:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.292908:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17330176, 17334272) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.292922:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17330176=0x1087000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.292984:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.292987:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17334272 00000020:00010000:1.0:1789760712.292996:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.292999:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.293003:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.293007:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.293010:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.293018:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.293024:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.293042:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760712.293161:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 17330176<>17334272 00000080:00200000:1.0:1789760712.351547:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4231, 4231] 00000080:00200000:1.0:1789760712.351564:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.351570:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.351594:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17334272 i_blocks 34823 00000020:00200000:1.0:1789760712.351598:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.351608:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.351642:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17334272, count: 4096 00000080:00200000:1.0:1789760712.351740:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17334272, bytes: 4096 00000080:00200000:1.0:1789760712.351752:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.351760:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.351762:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17334272, 17338368) 0 0 00000080:00200000:1.0:1789760712.351767:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.351783:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4232, 4232] 00020000:00200000:1.0:1789760712.351791:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17334272 chunk: [17334272, 17338368) 17338368, 4096 00020000:00200000:1.0:1789760712.351796:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.351814:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4232, 4232] 00000008:00010000:1.0:1789760712.351832:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.351837:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.351849:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17334272, 17338368) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.351863:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17334272=0x1088000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.351930:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.351933:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17338368 00000020:00010000:1.0:1789760712.351941:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.351945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.351948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.351951:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.351954:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.351962:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.351968:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.351985:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760712.352089:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 17334272<>17338368 00000080:00200000:1.0:1789760712.398169:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4232, 4232] 00000080:00200000:1.0:1789760712.398186:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.398192:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.398324:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17338368 i_blocks 34823 00000020:00200000:1.0:1789760712.398329:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.398339:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.398375:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17338368, count: 4096 00000080:00200000:1.0:1789760712.398421:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17338368, bytes: 4096 00000080:00200000:1.0:1789760712.398430:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.398438:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.398441:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17338368, 17342464) 0 0 00000080:00200000:1.0:1789760712.398445:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.398461:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4233, 4233] 00020000:00200000:1.0:1789760712.398469:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17338368 chunk: [17338368, 17342464) 17342464, 4096 00020000:00200000:1.0:1789760712.398473:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.398492:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4233, 4233] 00000008:00010000:1.0:1789760712.398509:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.398515:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.398525:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17338368, 17342464) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.398538:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17338368=0x1089000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.398982:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.398987:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17342464 00000020:00010000:1.0:1789760712.398997:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.399001:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.399005:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.399009:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.399012:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.399021:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.399039:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.399063:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760712.399955:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 17338368<>17342464 00000100:00100000:1.0:1789760712.400014:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760712.400039:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507169408:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760712.601099:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4236, 4236] 00000080:00200000:1.0:1789760712.601116:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.601122:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.601138:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17354752 i_blocks 34823 00000020:00200000:1.0:1789760712.601142:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.601153:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.601192:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17354752, count: 4096 00000080:00200000:1.0:1789760712.601208:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17354752, bytes: 4096 00000080:00200000:1.0:1789760712.601293:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.601302:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.601304:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17354752, 17358848) 0 0 00000080:00200000:1.0:1789760712.601310:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.601327:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4237, 4237] 00020000:00200000:1.0:1789760712.601336:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17354752 chunk: [17354752, 17358848) 17358848, 4096 00020000:00200000:1.0:1789760712.601340:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.601360:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4237, 4237] 00000008:00010000:1.0:1789760712.601416:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.601424:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.601436:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17354752, 17358848) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.601452:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17354752=0x108d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.601516:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.601519:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17358848 00000020:00010000:1.0:1789760712.601528:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.601532:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.601535:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.601539:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.601541:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.601549:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.601556:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.601573:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760712.601799:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 17354752<>17358848 00000100:00100000:1.0:1789760712.601843:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760712.601871:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507169920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760712.778491:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507170432:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760712.924100:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4244, 4244] 00000080:00200000:1.0:1789760712.924119:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760712.924125:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760712.924142:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17387520 i_blocks 34823 00000020:00200000:1.0:1789760712.924146:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760712.924156:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760712.924192:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17387520, count: 4096 00000080:00200000:1.0:1789760712.924209:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17387520, bytes: 4096 00000080:00200000:1.0:1789760712.924221:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760712.924230:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760712.924232:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17387520, 17391616) 0 0 00000080:00200000:1.0:1789760712.924237:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760712.924255:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4245, 4245] 00020000:00200000:1.0:1789760712.924263:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17387520 chunk: [17387520, 17391616) 17391616, 4096 00020000:00200000:1.0:1789760712.924268:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760712.924290:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4245, 4245] 00000008:00010000:1.0:1789760712.924309:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760712.924315:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760712.924327:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17387520, 17391616) append 0 append lockpos 0 00000080:00200000:1.0:1789760712.924341:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17387520=0x1095000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760712.925322:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760712.925329:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17391616 00000020:00010000:1.0:1789760712.925341:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be96c8 (ffff8d25c8038c68/0) at cl_lock_release():245 00000020:00010000:1.0:1789760712.925345:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be96c8 (ffff8d25c8038c68/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.925349:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760712.925353:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be96c8 (ffff8d25c8038c68/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760712.925357:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038c68/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760712.925373:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760712.925410:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760712.925441:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760712.925559:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 17387520<>17391616 00000100:00100000:1.0:1789760712.925601:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760712.925627:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507170944:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760713.569371:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507172480:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760713.599960:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760713.599978:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760713.599991:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760713.600013:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760713.600049:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507172480:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760713.627605:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971578 gen 3 00000008:00010000:1.0:1789760713.627660:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760713.627667:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760713.627677:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760713.627764:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760713.627795:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507172608:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760713.769275:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4261, 4261] 00000080:00200000:1.0:1789760713.769292:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760713.769298:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760713.769313:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17457152 i_blocks 34823 00000020:00200000:1.0:1789760713.769317:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760713.769327:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760713.769373:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17457152, count: 4096 00000080:00200000:1.0:1789760713.770521:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17457152, bytes: 4096 00000080:00200000:1.0:1789760713.770543:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760713.770553:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760713.770556:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17457152, 17461248) 0 0 00000080:00200000:1.0:1789760713.770561:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760713.770580:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4262, 4262] 00020000:00200000:1.0:1789760713.770597:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17457152 chunk: [17457152, 17461248) 17461248, 4096 00020000:00200000:1.0:1789760713.770601:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760713.770755:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4262, 4262] 00000008:00010000:1.0:1789760713.770775:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760713.770781:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760713.770794:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17457152, 17461248) append 0 append lockpos 0 00000080:00200000:1.0:1789760713.770814:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17457152=0x10a6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760713.770881:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760713.770884:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17461248 00000020:00010000:1.0:1789760713.770892:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760713.770896:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760713.770898:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760713.770901:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760713.770905:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760713.770914:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760713.770921:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760713.770946:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760713.771595:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 17457152<>17461248 00000100:00100000:1.0:1789760714.051564:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507174016:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760714.083004:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4269, 4269] 00000080:00200000:1.0:1789760714.083022:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760714.083028:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760714.083045:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17489920 i_blocks 34823 00000020:00200000:1.0:1789760714.083049:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760714.083060:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760714.083094:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17489920, count: 4096 00000080:00200000:1.0:1789760714.083112:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17489920, bytes: 4096 00000080:00200000:1.0:1789760714.083122:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760714.083132:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760714.083134:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17489920, 17494016) 0 0 00000080:00200000:1.0:1789760714.083139:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760714.083158:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4270, 4270] 00020000:00200000:1.0:1789760714.083167:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17489920 chunk: [17489920, 17494016) 17494016, 4096 00020000:00200000:1.0:1789760714.083171:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760714.083203:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4270, 4270] 00000008:00010000:1.0:1789760714.083367:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760714.083374:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760714.083420:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17489920, 17494016) append 0 append lockpos 0 00000080:00200000:1.0:1789760714.083435:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17489920=0x10ae000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760714.083492:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760714.083495:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17494016 00000020:00010000:1.0:1789760714.083504:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760714.083507:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760714.083509:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760714.083513:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760714.083516:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760714.083522:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760714.083530:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760714.083607:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760714.084997:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 17489920<>17494016 00000100:00100000:1.0:1789760714.195815:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507174400:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760714.423763:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4275, 4275] 00000080:00200000:1.0:1789760714.423777:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760714.423782:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760714.423794:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17514496 i_blocks 34823 00000020:00200000:1.0:1789760714.423798:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760714.423807:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760714.423834:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17514496, count: 4096 00000080:00200000:1.0:1789760714.423840:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17514496, bytes: 4096 00000080:00200000:1.0:1789760714.423848:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760714.423856:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760714.423858:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17514496, 17518592) 0 0 00000080:00200000:1.0:1789760714.423862:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760714.423875:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4276, 4276] 00020000:00200000:1.0:1789760714.423882:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17514496 chunk: [17514496, 17518592) 17518592, 4096 00020000:00200000:1.0:1789760714.423886:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760714.423910:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4276, 4276] 00000008:00010000:1.0:1789760714.423924:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760714.423929:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760714.423938:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17514496, 17518592) append 0 append lockpos 0 00000080:00200000:1.0:1789760714.423948:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17514496=0x10b4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760714.423991:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760714.423993:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17518592 00000020:00010000:1.0:1789760714.424000:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760714.424003:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760714.424006:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760714.424010:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760714.424012:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760714.424019:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760714.424024:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760714.424038:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760714.424239:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 17514496<>17518592 00000100:00100000:1.0:1789760714.424283:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760714.424299:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507174912:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760714.860797:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507175808:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760715.536893:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507176832:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760715.584891:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4291, 4291] 00000080:00200000:1.0:1789760715.584914:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760715.584922:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760715.584933:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17580032 i_blocks 34823 00000020:00200000:1.0:1789760715.584937:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760715.584945:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760715.584970:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17580032, count: 4096 00000080:00200000:1.0:1789760715.584976:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17580032, bytes: 4096 00000080:00200000:1.0:1789760715.584984:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760715.584991:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760715.584994:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17580032, 17584128) 0 0 00000080:00200000:1.0:1789760715.584998:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760715.585011:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4292, 4292] 00020000:00200000:1.0:1789760715.585017:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17580032 chunk: [17580032, 17584128) 17584128, 4096 00020000:00200000:1.0:1789760715.585021:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760715.585035:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4292, 4292] 00000008:00010000:1.0:1789760715.585049:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760715.585054:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760715.585062:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17580032, 17584128) append 0 append lockpos 0 00000080:00200000:1.0:1789760715.585073:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17580032=0x10c4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760715.585116:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760715.585119:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17584128 00000020:00010000:1.0:1789760715.585126:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760715.585129:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.585131:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.585135:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760715.585138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760715.585144:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760715.585150:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760715.585163:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760715.585265:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 17580032<>17584128 00000080:00200000:1.0:1789760715.632447:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4292, 4292] 00000080:00200000:1.0:1789760715.632461:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760715.632466:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760715.632479:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17584128 i_blocks 34823 00000020:00200000:1.0:1789760715.632482:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760715.632490:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760715.632517:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17584128, count: 4096 00000080:00200000:1.0:1789760715.632523:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17584128, bytes: 4096 00000080:00200000:1.0:1789760715.632531:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760715.632538:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760715.632540:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17584128, 17588224) 0 0 00000080:00200000:1.0:1789760715.632544:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760715.632556:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4293, 4293] 00020000:00200000:1.0:1789760715.632562:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17584128 chunk: [17584128, 17588224) 17588224, 4096 00020000:00200000:1.0:1789760715.632565:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760715.632580:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4293, 4293] 00000008:00010000:1.0:1789760715.632593:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760715.632598:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760715.632617:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17584128, 17588224) append 0 append lockpos 0 00000080:00200000:1.0:1789760715.632733:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17584128=0x10c5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760715.632780:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760715.632783:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17588224 00000020:00010000:1.0:1789760715.632790:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760715.632793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.632795:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.632798:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760715.632801:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760715.632807:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760715.632813:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760715.632827:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760715.632903:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 17584128<>17588224 00000080:00200000:1.0:1789760715.688245:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4293, 4293] 00000080:00200000:1.0:1789760715.688259:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760715.688265:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760715.688276:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17588224 i_blocks 34823 00000020:00200000:1.0:1789760715.688279:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760715.688287:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760715.688312:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17588224, count: 4096 00000080:00200000:1.0:1789760715.688318:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17588224, bytes: 4096 00000080:00200000:1.0:1789760715.688326:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760715.688333:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760715.688335:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17588224, 17592320) 0 0 00000080:00200000:1.0:1789760715.688340:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760715.688353:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4294, 4294] 00020000:00200000:1.0:1789760715.688360:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17588224 chunk: [17588224, 17592320) 17592320, 4096 00020000:00200000:1.0:1789760715.688364:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760715.688407:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4294, 4294] 00000008:00010000:1.0:1789760715.688421:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760715.688425:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760715.688434:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17588224, 17592320) append 0 append lockpos 0 00000080:00200000:1.0:1789760715.688445:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17588224=0x10c6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760715.688487:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760715.688490:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17592320 00000020:00010000:1.0:1789760715.688496:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760715.688499:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.688502:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.688505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760715.688508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760715.688513:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760715.688519:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760715.688531:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760715.688603:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 17588224<>17592320 00000080:00200000:1.0:1789760715.761796:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4294, 4294] 00000080:00200000:1.0:1789760715.761810:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760715.761816:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760715.761827:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17592320 i_blocks 34823 00000020:00200000:1.0:1789760715.761831:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760715.761838:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760715.761864:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17592320, count: 4096 00000080:00200000:1.0:1789760715.761869:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17592320, bytes: 4096 00000080:00200000:1.0:1789760715.761876:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760715.761884:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760715.761886:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17592320, 17596416) 0 0 00000080:00200000:1.0:1789760715.761889:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760715.761914:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4295, 4295] 00020000:00200000:1.0:1789760715.761922:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17592320 chunk: [17592320, 17596416) 17596416, 4096 00020000:00200000:1.0:1789760715.761926:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760715.761941:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4295, 4295] 00000008:00010000:1.0:1789760715.761954:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760715.761958:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760715.761967:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17592320, 17596416) append 0 append lockpos 0 00000080:00200000:1.0:1789760715.761978:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17592320=0x10c7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760715.762021:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760715.762024:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17596416 00000020:00010000:1.0:1789760715.762031:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760715.762034:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.762037:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.762040:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760715.762043:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760715.762049:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760715.762054:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760715.762068:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760715.762143:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 17592320<>17596416 00000080:00200000:1.0:1789760715.813065:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4295, 4295] 00000080:00200000:1.0:1789760715.813078:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760715.813083:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760715.813094:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17596416 i_blocks 34823 00000020:00200000:1.0:1789760715.813097:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760715.813104:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760715.813130:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17596416, count: 4096 00000080:00200000:1.0:1789760715.813135:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17596416, bytes: 4096 00000080:00200000:1.0:1789760715.813143:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760715.813150:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760715.813152:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17596416, 17600512) 0 0 00000080:00200000:1.0:1789760715.813156:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760715.813169:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4296, 4296] 00020000:00200000:1.0:1789760715.813186:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17596416 chunk: [17596416, 17600512) 17600512, 4096 00020000:00200000:1.0:1789760715.813190:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760715.813206:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4296, 4296] 00000008:00010000:1.0:1789760715.813231:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760715.813237:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760715.813245:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17596416, 17600512) append 0 append lockpos 0 00000080:00200000:1.0:1789760715.813256:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17596416=0x10c8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760715.813298:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760715.813301:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17600512 00000020:00010000:1.0:1789760715.813308:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760715.813311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.813313:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760715.813317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760715.813319:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760715.813325:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760715.813330:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760715.813344:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760715.813750:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 17596416<>17600512 00000100:00100000:1.0:1789760715.813788:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760715.813802:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507177472:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760716.491881:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4304, 4304] 00000080:00200000:1.0:1789760716.491895:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760716.491901:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760716.491913:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17633280 i_blocks 34823 00000020:00200000:1.0:1789760716.491916:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760716.491925:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760716.491953:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17633280, count: 4096 00000080:00200000:1.0:1789760716.491960:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17633280, bytes: 4096 00000080:00200000:1.0:1789760716.491967:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760716.491974:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760716.491976:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17633280, 17637376) 0 0 00000080:00200000:1.0:1789760716.491980:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760716.491993:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4305, 4305] 00020000:00200000:1.0:1789760716.492000:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17633280 chunk: [17633280, 17637376) 17637376, 4096 00020000:00200000:1.0:1789760716.492003:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760716.492018:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4305, 4305] 00000008:00010000:1.0:1789760716.492031:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760716.492036:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760716.492044:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17633280, 17637376) append 0 append lockpos 0 00000080:00200000:1.0:1789760716.492055:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17633280=0x10d1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760716.492094:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.492096:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17637376 00000020:00010000:1.0:1789760716.492102:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760716.492105:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.492107:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.492110:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760716.492112:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760716.492118:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760716.492123:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760716.492138:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760716.492589:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 17633280<>17637376 00000100:00100000:1.0:1789760716.492874:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760716.492891:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507178624:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760716.593491:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4305, 4305] 00000080:00200000:1.0:1789760716.593505:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760716.593510:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760716.593531:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17637376 i_blocks 34823 00000020:00200000:1.0:1789760716.593535:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760716.593542:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760716.593566:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17637376, count: 4096 00000080:00200000:1.0:1789760716.593571:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17637376, bytes: 4096 00000080:00200000:1.0:1789760716.593578:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760716.593585:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760716.593587:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17637376, 17641472) 0 0 00000080:00200000:1.0:1789760716.593591:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760716.593604:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4306, 4306] 00020000:00200000:1.0:1789760716.593610:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17637376 chunk: [17637376, 17641472) 17641472, 4096 00020000:00200000:1.0:1789760716.593614:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760716.593627:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4306, 4306] 00000008:00010000:1.0:1789760716.593641:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760716.593645:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760716.593653:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17637376, 17641472) append 0 append lockpos 0 00000080:00200000:1.0:1789760716.593663:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17637376=0x10d2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760716.593709:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.593712:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17641472 00000020:00010000:1.0:1789760716.593718:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760716.593721:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.593724:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.593727:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760716.593730:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760716.593736:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760716.593742:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.672541:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4306, 4306] 00000080:00200000:1.0:1789760716.672554:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760716.672559:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760716.672570:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17641472 i_blocks 34823 00000020:00200000:1.0:1789760716.672573:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760716.672581:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760716.672605:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17641472, count: 4096 00000080:00200000:1.0:1789760716.672611:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17641472, bytes: 4096 00000080:00200000:1.0:1789760716.672628:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760716.672636:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760716.672638:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17641472, 17645568) 0 0 00000080:00200000:1.0:1789760716.672643:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760716.672656:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4307, 4307] 00020000:00200000:1.0:1789760716.672662:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17641472 chunk: [17641472, 17645568) 17645568, 4096 00020000:00200000:1.0:1789760716.672666:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760716.672681:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4307, 4307] 00000008:00010000:1.0:1789760716.672694:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760716.672698:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760716.672707:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17641472, 17645568) append 0 append lockpos 0 00000080:00200000:1.0:1789760716.672718:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17641472=0x10d3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760716.672762:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.672765:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17645568 00000020:00010000:1.0:1789760716.672771:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760716.672775:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.672778:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.672781:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760716.672784:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760716.672790:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760716.672795:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760716.672809:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760716.672882:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 17641472<>17645568 00000080:00200000:1.0:1789760716.744818:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4307, 4307] 00000080:00200000:1.0:1789760716.744832:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760716.744837:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760716.744848:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17645568 i_blocks 34823 00000020:00200000:1.0:1789760716.744852:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760716.744860:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760716.744887:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17645568, count: 4096 00000080:00200000:1.0:1789760716.744893:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17645568, bytes: 4096 00000080:00200000:1.0:1789760716.744902:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760716.744909:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760716.744911:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17645568, 17649664) 0 0 00000080:00200000:1.0:1789760716.744915:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760716.744930:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4308, 4308] 00020000:00200000:1.0:1789760716.744937:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17645568 chunk: [17645568, 17649664) 17649664, 4096 00020000:00200000:1.0:1789760716.744941:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760716.744957:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4308, 4308] 00000008:00010000:1.0:1789760716.744971:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760716.744975:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760716.744984:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17645568, 17649664) append 0 append lockpos 0 00000080:00200000:1.0:1789760716.744996:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17645568=0x10d4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760716.745043:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.745045:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17649664 00000020:00010000:1.0:1789760716.745052:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760716.745054:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.745057:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.745060:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760716.745063:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760716.745069:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760716.745075:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760716.745089:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760716.745177:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 17645568<>17649664 00000080:00200000:1.0:1789760716.826730:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4308, 4308] 00000080:00200000:1.0:1789760716.826746:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760716.826751:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760716.826764:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17649664 i_blocks 34823 00000020:00200000:1.0:1789760716.826767:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760716.826784:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760716.826809:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17649664, count: 4096 00000080:00200000:1.0:1789760716.826815:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17649664, bytes: 4096 00000080:00200000:1.0:1789760716.826823:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760716.826829:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760716.826831:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17649664, 17653760) 0 0 00000080:00200000:1.0:1789760716.826835:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760716.826847:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4309, 4309] 00020000:00200000:1.0:1789760716.826854:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17649664 chunk: [17649664, 17653760) 17653760, 4096 00020000:00200000:1.0:1789760716.826857:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760716.826871:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4309, 4309] 00000008:00010000:1.0:1789760716.826882:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760716.826886:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760716.826895:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17649664, 17653760) append 0 append lockpos 0 00000080:00200000:1.0:1789760716.826905:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17649664=0x10d5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760716.826940:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.826942:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17653760 00000020:00010000:1.0:1789760716.826948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760716.826951:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.826954:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.826957:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760716.826959:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760716.826965:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760716.826971:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.937901:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4309, 4309] 00000080:00200000:1.0:1789760716.937915:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760716.937920:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760716.937932:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17653760 i_blocks 34823 00000020:00200000:1.0:1789760716.937935:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760716.937944:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760716.937969:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17653760, count: 4096 00000080:00200000:1.0:1789760716.937975:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17653760, bytes: 4096 00000080:00200000:1.0:1789760716.937982:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760716.937989:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760716.937991:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17653760, 17657856) 0 0 00000080:00200000:1.0:1789760716.937995:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760716.938005:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4310, 4310] 00020000:00200000:1.0:1789760716.938014:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17653760 chunk: [17653760, 17657856) 17657856, 4096 00020000:00200000:1.0:1789760716.938018:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760716.938032:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4310, 4310] 00000008:00010000:1.0:1789760716.938046:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760716.938050:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760716.938059:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17653760, 17657856) append 0 append lockpos 0 00000080:00200000:1.0:1789760716.938071:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17653760=0x10d6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760716.938121:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760716.938124:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17657856 00000020:00010000:1.0:1789760716.938130:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760716.938134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.938136:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760716.938140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760716.938142:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760716.938148:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760716.938153:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760716.938243:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760716.938547:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 17653760<>17657856 00000080:00200000:1.0:1789760717.537275:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4318, 4318] 00000080:00200000:1.0:1789760717.537289:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760717.537294:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760717.537306:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17690624 i_blocks 34823 00000020:00200000:1.0:1789760717.537309:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760717.537317:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760717.537340:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17690624, count: 4096 00000080:00200000:1.0:1789760717.537346:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17690624, bytes: 4096 00000080:00200000:1.0:1789760717.537354:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760717.537361:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760717.537364:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17690624, 17694720) 0 0 00000080:00200000:1.0:1789760717.537367:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760717.537408:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4319, 4319] 00020000:00200000:1.0:1789760717.537416:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17690624 chunk: [17690624, 17694720) 17694720, 4096 00020000:00200000:1.0:1789760717.537420:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760717.537437:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4319, 4319] 00000008:00010000:1.0:1789760717.537450:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760717.537455:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760717.537464:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17690624, 17694720) append 0 append lockpos 0 00000080:00200000:1.0:1789760717.537474:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17690624=0x10df000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760717.537521:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760717.537524:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17694720 00000020:00010000:1.0:1789760717.537531:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760717.537534:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.537537:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.537540:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760717.537543:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760717.537549:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760717.537554:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760717.537571:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760717.537712:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 17690624<>17694720 00000080:00200000:1.0:1789760717.583789:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4319, 4319] 00000080:00200000:1.0:1789760717.583806:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760717.583811:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760717.583822:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17694720 i_blocks 34823 00000020:00200000:1.0:1789760717.583826:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760717.583833:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760717.583857:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17694720, count: 4096 00000080:00200000:1.0:1789760717.583863:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17694720, bytes: 4096 00000080:00200000:1.0:1789760717.583870:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760717.583876:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760717.583878:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17694720, 17698816) 0 0 00000080:00200000:1.0:1789760717.583882:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760717.583893:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4320, 4320] 00020000:00200000:1.0:1789760717.583912:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17694720 chunk: [17694720, 17698816) 17698816, 4096 00020000:00200000:1.0:1789760717.583916:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760717.583930:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4320, 4320] 00000008:00010000:1.0:1789760717.583943:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760717.583948:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760717.583956:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17694720, 17698816) append 0 append lockpos 0 00000080:00200000:1.0:1789760717.583966:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17694720=0x10e0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760717.584002:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760717.584005:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17698816 00000020:00010000:1.0:1789760717.584010:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760717.584013:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.584016:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.584019:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760717.584022:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760717.584027:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760717.584031:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760717.584043:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760717.584108:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 17694720<>17698816 00000080:00200000:1.0:1789760717.653285:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4320, 4320] 00000080:00200000:1.0:1789760717.653299:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760717.653305:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760717.653318:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17698816 i_blocks 34823 00000020:00200000:1.0:1789760717.653321:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760717.653329:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760717.653354:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17698816, count: 4096 00000080:00200000:1.0:1789760717.653360:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17698816, bytes: 4096 00000080:00200000:1.0:1789760717.653368:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760717.653405:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760717.653408:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17698816, 17702912) 0 0 00000080:00200000:1.0:1789760717.653412:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760717.653424:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4321, 4321] 00020000:00200000:1.0:1789760717.653431:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17698816 chunk: [17698816, 17702912) 17702912, 4096 00020000:00200000:1.0:1789760717.653434:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760717.653460:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4321, 4321] 00000008:00010000:1.0:1789760717.653474:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760717.653479:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760717.653489:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17698816, 17702912) append 0 append lockpos 0 00000080:00200000:1.0:1789760717.653499:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17698816=0x10e1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760717.653546:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760717.653549:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17702912 00000020:00010000:1.0:1789760717.653557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760717.653560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.653563:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.653567:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760717.653570:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760717.653576:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760717.653581:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760717.653598:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760717.653696:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 17698816<>17702912 00000100:00100000:1.0:1789760717.653742:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507181312:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760717.746409:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4321, 4321] 00000080:00200000:1.0:1789760717.746423:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760717.746429:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760717.746441:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17702912 i_blocks 34823 00000020:00200000:1.0:1789760717.746444:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760717.746453:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760717.746480:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17702912, count: 4096 00000080:00200000:1.0:1789760717.746486:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17702912, bytes: 4096 00000080:00200000:1.0:1789760717.746494:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760717.746501:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760717.746503:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17702912, 17707008) 0 0 00000080:00200000:1.0:1789760717.746507:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760717.746520:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4322, 4322] 00020000:00200000:1.0:1789760717.746526:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17702912 chunk: [17702912, 17707008) 17707008, 4096 00020000:00200000:1.0:1789760717.746531:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760717.746545:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4322, 4322] 00000008:00010000:1.0:1789760717.746558:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760717.746563:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760717.746571:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17702912, 17707008) append 0 append lockpos 0 00000080:00200000:1.0:1789760717.746581:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17702912=0x10e2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760717.746623:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760717.746625:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17707008 00000020:00010000:1.0:1789760717.746631:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760717.746635:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.746638:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.746641:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760717.746644:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760717.746650:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760717.746655:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760717.746670:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760717.746744:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 17702912<>17707008 00000080:00200000:1.0:1789760717.830786:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4322, 4322] 00000080:00200000:1.0:1789760717.830800:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760717.830805:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760717.830816:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17707008 i_blocks 34823 00000020:00200000:1.0:1789760717.830819:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760717.830827:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760717.830852:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17707008, count: 4096 00000080:00200000:1.0:1789760717.830857:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17707008, bytes: 4096 00000080:00200000:1.0:1789760717.830864:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760717.830872:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760717.830874:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17707008, 17711104) 0 0 00000080:00200000:1.0:1789760717.830877:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760717.830889:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4323, 4323] 00020000:00200000:1.0:1789760717.830896:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17707008 chunk: [17707008, 17711104) 17711104, 4096 00020000:00200000:1.0:1789760717.830899:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760717.830913:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4323, 4323] 00000008:00010000:1.0:1789760717.830927:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760717.830931:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760717.830940:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17707008, 17711104) append 0 append lockpos 0 00000080:00200000:1.0:1789760717.830950:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17707008=0x10e3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760717.830991:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760717.830993:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17711104 00000020:00010000:1.0:1789760717.830999:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760717.831002:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.831005:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.831008:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760717.831010:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760717.831016:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760717.831021:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760717.831034:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760717.831109:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 17707008<>17711104 00000080:00200000:1.0:1789760717.884496:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4323, 4323] 00000080:00200000:1.0:1789760717.884509:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760717.884514:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760717.884525:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17711104 i_blocks 34823 00000020:00200000:1.0:1789760717.884528:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760717.884536:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760717.884564:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17711104, count: 4096 00000080:00200000:1.0:1789760717.884570:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17711104, bytes: 4096 00000080:00200000:1.0:1789760717.884579:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760717.884586:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760717.884589:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17711104, 17715200) 0 0 00000080:00200000:1.0:1789760717.884593:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760717.884606:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4324, 4324] 00020000:00200000:1.0:1789760717.884684:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17711104 chunk: [17711104, 17715200) 17715200, 4096 00020000:00200000:1.0:1789760717.884689:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760717.884706:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4324, 4324] 00000008:00010000:1.0:1789760717.884721:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760717.884725:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760717.884733:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17711104, 17715200) append 0 append lockpos 0 00000080:00200000:1.0:1789760717.884744:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17711104=0x10e4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760717.884789:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760717.884792:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17715200 00000020:00010000:1.0:1789760717.884799:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760717.884802:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.884805:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.884808:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760717.884811:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760717.884818:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760717.884823:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760717.884837:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760717.885000:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 17711104<>17715200 00000080:00200000:1.0:1789760717.947040:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4324, 4324] 00000080:00200000:1.0:1789760717.947052:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760717.947057:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760717.947069:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17715200 i_blocks 34823 00000020:00200000:1.0:1789760717.947072:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760717.947078:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760717.947101:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17715200, count: 4096 00000080:00200000:1.0:1789760717.947106:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17715200, bytes: 4096 00000080:00200000:1.0:1789760717.947113:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760717.947120:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760717.947122:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17715200, 17719296) 0 0 00000080:00200000:1.0:1789760717.947134:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760717.947146:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4325, 4325] 00020000:00200000:1.0:1789760717.947159:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17715200 chunk: [17715200, 17719296) 17719296, 4096 00020000:00200000:1.0:1789760717.947169:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760717.947182:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4325, 4325] 00000008:00010000:1.0:1789760717.947194:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760717.947206:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760717.947214:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17715200, 17719296) append 0 append lockpos 0 00000080:00200000:1.0:1789760717.947224:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17715200=0x10e5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760717.947260:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760717.947263:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17719296 00000020:00010000:1.0:1789760717.947269:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760717.947272:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.947274:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760717.947277:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760717.947280:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760717.947285:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760717.947290:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760717.947303:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760717.947371:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 17715200<>17719296 00000100:00100000:1.0:1789760717.947585:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507181824:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760718.794401:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4337, 4337] 00000080:00200000:1.0:1789760718.794414:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760718.794419:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760718.794431:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17768448 i_blocks 34823 00000020:00200000:1.0:1789760718.794434:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760718.794443:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760718.794470:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17768448, count: 4096 00000080:00200000:1.0:1789760718.794475:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17768448, bytes: 4096 00000080:00200000:1.0:1789760718.794487:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760718.794496:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760718.794498:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17768448, 17772544) 0 0 00000080:00200000:1.0:1789760718.794502:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760718.794516:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4338, 4338] 00020000:00200000:1.0:1789760718.794524:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17768448 chunk: [17768448, 17772544) 17772544, 4096 00020000:00200000:1.0:1789760718.794528:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760718.794544:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4338, 4338] 00000008:00010000:1.0:1789760718.794561:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760718.794565:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760718.794574:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17768448, 17772544) append 0 append lockpos 0 00000080:00200000:1.0:1789760718.794586:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17768448=0x10f2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760718.794644:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760718.794647:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17772544 00000020:00010000:1.0:1789760718.794653:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760718.794656:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760718.794659:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760718.794662:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760718.794664:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760718.794671:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760718.794676:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760718.794689:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760718.794776:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 17768448<>17772544 00000100:00100000:1.0:1789760718.794815:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760718.794828:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507183488:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760718.880049:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4338, 4338] 00000080:00200000:1.0:1789760718.880064:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760718.880071:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760718.880087:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17772544 i_blocks 34823 00000020:00200000:1.0:1789760718.880090:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760718.880101:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760718.880135:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17772544, count: 4096 00000080:00200000:1.0:1789760718.880143:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17772544, bytes: 4096 00000080:00200000:1.0:1789760718.880162:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760718.880172:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760718.880174:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17772544, 17776640) 0 0 00000080:00200000:1.0:1789760718.880179:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760718.880204:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4339, 4339] 00020000:00200000:1.0:1789760718.880214:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17772544 chunk: [17772544, 17776640) 17776640, 4096 00020000:00200000:1.0:1789760718.880219:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760718.880238:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4339, 4339] 00000008:00010000:1.0:1789760718.880256:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760718.880262:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760718.880274:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17772544, 17776640) append 0 append lockpos 0 00000080:00200000:1.0:1789760718.880292:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17772544=0x10f3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760718.880351:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760718.880354:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17776640 00000020:00010000:1.0:1789760718.880362:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760718.880366:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760718.880369:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760718.880373:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760718.880417:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760718.880426:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760718.880434:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760718.880450:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760718.880546:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 17772544<>17776640 00000080:00200000:1.0:1789760718.910514:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4339, 4339] 00000080:00200000:1.0:1789760718.910645:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760718.910652:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760718.910669:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17776640 i_blocks 34823 00000020:00200000:1.0:1789760718.910673:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760718.910761:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760718.910799:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17776640, count: 4096 00000080:00200000:1.0:1789760718.910807:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17776640, bytes: 4096 00000080:00200000:1.0:1789760718.910819:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760718.910829:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760718.910832:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17776640, 17780736) 0 0 00000080:00200000:1.0:1789760718.910837:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760718.910854:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4340, 4340] 00020000:00200000:1.0:1789760718.910862:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17776640 chunk: [17776640, 17780736) 17780736, 4096 00020000:00200000:1.0:1789760718.910867:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760718.910889:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4340, 4340] 00000008:00010000:1.0:1789760718.910967:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760718.910974:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760718.910987:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17776640, 17780736) append 0 append lockpos 0 00000080:00200000:1.0:1789760718.911001:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17776640=0x10f4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760718.911064:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760718.911067:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17780736 00000020:00010000:1.0:1789760718.911075:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760718.911079:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760718.911082:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760718.911085:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760718.911088:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760718.911097:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760718.911104:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760718.911126:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760718.912124:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 17776640<>17780736 00000100:00100000:1.0:1789760718.912187:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760718.912211:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507183744:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760720.282125:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760720.282164:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507186560:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760720.412863:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4364, 4364] 00000080:00200000:1.0:1789760720.412880:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760720.412886:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760720.412943:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17879040 i_blocks 36871 00000020:00200000:1.0:1789760720.412948:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760720.412958:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760720.412991:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17879040, count: 4096 00000080:00200000:1.0:1789760720.412998:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17879040, bytes: 4096 00000080:00200000:1.0:1789760720.413012:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760720.413021:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760720.413023:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17879040, 17883136) 0 0 00000080:00200000:1.0:1789760720.413026:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760720.413042:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4365, 4365] 00020000:00200000:1.0:1789760720.413049:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17879040 chunk: [17879040, 17883136) 17883136, 4096 00020000:00200000:1.0:1789760720.413053:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760720.413072:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4365, 4365] 00000008:00010000:1.0:1789760720.413090:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760720.413097:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760720.413108:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17879040, 17883136) append 0 append lockpos 0 00000080:00200000:1.0:1789760720.413121:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17879040=0x110d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760720.413258:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760720.413262:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17883136 00000020:00010000:1.0:1789760720.413271:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760720.413275:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.413279:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.413282:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760720.413285:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760720.413293:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760720.413302:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760720.413320:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760720.413525:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 17879040<>17883136 00000080:00200000:1.0:1789760720.442206:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4365, 4365] 00000080:00200000:1.0:1789760720.442223:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760720.442229:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760720.442245:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17883136 i_blocks 36871 00000020:00200000:1.0:1789760720.442248:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760720.442258:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760720.442297:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17883136, count: 4096 00000080:00200000:1.0:1789760720.442304:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17883136, bytes: 4096 00000080:00200000:1.0:1789760720.442314:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760720.442323:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760720.442325:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17883136, 17887232) 0 0 00000080:00200000:1.0:1789760720.442330:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760720.442347:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4366, 4366] 00020000:00200000:1.0:1789760720.442356:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17883136 chunk: [17883136, 17887232) 17887232, 4096 00020000:00200000:1.0:1789760720.442361:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760720.442413:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4366, 4366] 00000008:00010000:1.0:1789760720.442432:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760720.442438:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760720.442449:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17883136, 17887232) append 0 append lockpos 0 00000080:00200000:1.0:1789760720.442464:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17883136=0x110e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760720.442518:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760720.442521:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17887232 00000020:00010000:1.0:1789760720.442529:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760720.442532:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.442535:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.442539:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760720.442542:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760720.442550:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760720.442557:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760720.442574:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760720.442744:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 17883136<>17887232 00000080:00200000:1.0:1789760720.486434:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4366, 4366] 00000080:00200000:1.0:1789760720.486451:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760720.486457:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760720.486473:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17887232 i_blocks 36871 00000020:00200000:1.0:1789760720.486476:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760720.486487:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760720.486519:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17887232, count: 4096 00000080:00200000:1.0:1789760720.486526:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17887232, bytes: 4096 00000080:00200000:1.0:1789760720.486536:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760720.486545:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760720.486547:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17887232, 17891328) 0 0 00000080:00200000:1.0:1789760720.486551:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760720.486568:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4367, 4367] 00020000:00200000:1.0:1789760720.486578:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17887232 chunk: [17887232, 17891328) 17891328, 4096 00020000:00200000:1.0:1789760720.486583:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760720.486672:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4367, 4367] 00000008:00010000:1.0:1789760720.486693:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760720.486698:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760720.486711:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17887232, 17891328) append 0 append lockpos 0 00000080:00200000:1.0:1789760720.486725:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17887232=0x110f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760720.486861:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760720.486864:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17891328 00000020:00010000:1.0:1789760720.486873:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760720.486877:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.486880:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.486884:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760720.486888:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760720.487052:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760720.487060:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760720.487080:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760720.487356:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 17887232<>17891328 00000080:00200000:1.0:1789760720.548855:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4367, 4367] 00000080:00200000:1.0:1789760720.548876:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760720.548883:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760720.548912:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17891328 i_blocks 36871 00000020:00200000:1.0:1789760720.548917:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760720.548929:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760720.548966:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17891328, count: 4096 00000080:00200000:1.0:1789760720.548974:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17891328, bytes: 4096 00000080:00200000:1.0:1789760720.548986:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760720.548994:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760720.548997:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17891328, 17895424) 0 0 00000080:00200000:1.0:1789760720.549001:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760720.549017:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4368, 4368] 00020000:00200000:1.0:1789760720.549025:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17891328 chunk: [17891328, 17895424) 17895424, 4096 00020000:00200000:1.0:1789760720.549030:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760720.549049:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4368, 4368] 00000008:00010000:1.0:1789760720.549069:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760720.549076:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760720.549088:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17891328, 17895424) append 0 append lockpos 0 00000080:00200000:1.0:1789760720.549104:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17891328=0x1110000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760720.549190:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760720.549194:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17895424 00000020:00010000:1.0:1789760720.549204:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760720.549208:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.549211:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.549214:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760720.549217:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760720.549226:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760720.549233:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760720.549250:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760720.549357:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 17891328<>17895424 00000080:00200000:1.0:1789760720.590061:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4368, 4368] 00000080:00200000:1.0:1789760720.590080:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760720.590087:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760720.590102:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17895424 i_blocks 36871 00000020:00200000:1.0:1789760720.590106:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760720.590116:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760720.590163:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17895424, count: 4096 00000080:00200000:1.0:1789760720.590170:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17895424, bytes: 4096 00000080:00200000:1.0:1789760720.590193:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760720.590203:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760720.590205:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17895424, 17899520) 0 0 00000080:00200000:1.0:1789760720.590209:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760720.590226:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4369, 4369] 00020000:00200000:1.0:1789760720.590234:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17895424 chunk: [17895424, 17899520) 17899520, 4096 00020000:00200000:1.0:1789760720.590239:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760720.590257:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4369, 4369] 00000008:00010000:1.0:1789760720.590275:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760720.590281:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760720.590292:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17895424, 17899520) append 0 append lockpos 0 00000080:00200000:1.0:1789760720.590306:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17895424=0x1111000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760720.590362:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760720.590553:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17899520 00000020:00010000:1.0:1789760720.590563:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760720.590567:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.590570:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.590574:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760720.590670:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760720.590680:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760720.590686:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760720.590704:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760720.590861:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 17895424<>17899520 00000080:00200000:1.0:1789760720.663250:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4369, 4369] 00000080:00200000:1.0:1789760720.663409:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760720.663417:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760720.663435:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17899520 i_blocks 36871 00000020:00200000:1.0:1789760720.663441:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760720.663454:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760720.663489:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17899520, count: 4096 00000080:00200000:1.0:1789760720.663498:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17899520, bytes: 4096 00000080:00200000:1.0:1789760720.663512:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760720.663521:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760720.663523:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17899520, 17903616) 0 0 00000080:00200000:1.0:1789760720.663529:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760720.663547:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4370, 4370] 00020000:00200000:1.0:1789760720.663556:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17899520 chunk: [17899520, 17903616) 17903616, 4096 00020000:00200000:1.0:1789760720.663561:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760720.666328:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4370, 4370] 00000008:00010000:1.0:1789760720.667707:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760720.667721:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760720.667746:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17899520, 17903616) append 0 append lockpos 0 00000080:00200000:1.0:1789760720.667766:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17899520=0x1112000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760720.667854:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760720.667858:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17903616 00000020:00010000:1.0:1789760720.667868:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760720.667872:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.667875:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760720.667879:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760720.667882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760720.667894:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760720.667904:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760720.667938:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760720.669399:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 17899520<>17903616 00000100:00100000:1.0:1789760721.118749:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760721.118839:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507188608:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760721.186843:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4378, 4378] 00000080:00200000:1.0:1789760721.186881:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.186889:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.186916:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17936384 i_blocks 36871 00000020:00200000:1.0:1789760721.186922:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.186940:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.186974:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17936384, count: 4096 00000080:00200000:1.0:1789760721.186983:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17936384, bytes: 4096 00000080:00200000:1.0:1789760721.186999:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.187009:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.187012:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17936384, 17940480) 0 0 00000080:00200000:1.0:1789760721.187016:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.187033:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4379, 4379] 00020000:00200000:1.0:1789760721.187042:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17936384 chunk: [17936384, 17940480) 17940480, 4096 00020000:00200000:1.0:1789760721.187047:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.187070:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4379, 4379] 00000008:00010000:1.0:1789760721.187090:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.187098:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.187109:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17936384, 17940480) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.187124:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17936384=0x111b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.187372:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.187421:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17940480 00000020:00010000:1.0:1789760721.187431:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.187435:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.187438:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.187442:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.187445:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.187455:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.187462:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.187485:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760721.187672:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 17936384<>17940480 00000100:00100000:1.0:1789760721.187721:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760721.187749:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507188736:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760721.295316:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4379, 4379] 00000080:00200000:1.0:1789760721.295333:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.295340:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.295356:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17940480 i_blocks 36871 00000020:00200000:1.0:1789760721.295360:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.295369:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.295437:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17940480, count: 4096 00000080:00200000:1.0:1789760721.295446:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17940480, bytes: 4096 00000080:00200000:1.0:1789760721.295455:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.295465:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.295467:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17940480, 17944576) 0 0 00000080:00200000:1.0:1789760721.295471:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.295487:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4380, 4380] 00020000:00200000:1.0:1789760721.295496:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17940480 chunk: [17940480, 17944576) 17944576, 4096 00020000:00200000:1.0:1789760721.295502:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.295522:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4380, 4380] 00000008:00010000:1.0:1789760721.295540:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.295546:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.295557:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17940480, 17944576) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.295571:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17940480=0x111c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.295640:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.295644:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17944576 00000020:00010000:1.0:1789760721.295652:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.295655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.295659:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.295677:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.295680:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.295688:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.295695:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.295718:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760721.295825:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 17940480<>17944576 00000080:00200000:1.0:1789760721.379157:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4380, 4380] 00000080:00200000:1.0:1789760721.379174:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.379180:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.379434:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17944576 i_blocks 36871 00000020:00200000:1.0:1789760721.379440:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.379450:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.379485:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17944576, count: 4096 00000080:00200000:1.0:1789760721.379493:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17944576, bytes: 4096 00000080:00200000:1.0:1789760721.379511:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.379521:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.379523:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17944576, 17948672) 0 0 00000080:00200000:1.0:1789760721.379527:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.379544:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4381, 4381] 00020000:00200000:1.0:1789760721.379553:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17944576 chunk: [17944576, 17948672) 17948672, 4096 00020000:00200000:1.0:1789760721.379560:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.379592:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4381, 4381] 00000008:00010000:1.0:1789760721.379612:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.379618:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.379630:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17944576, 17948672) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.379645:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17944576=0x111d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.379712:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.379715:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17948672 00000020:00010000:1.0:1789760721.379726:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.379729:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.379732:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.379736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.379739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.379747:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.379754:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.379773:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760721.380008:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 17944576<>17948672 00000080:00200000:1.0:1789760721.428953:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4381, 4381] 00000080:00200000:1.0:1789760721.428970:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.428976:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.428991:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17948672 i_blocks 36871 00000020:00200000:1.0:1789760721.428994:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.429005:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.429038:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17948672, count: 4096 00000080:00200000:1.0:1789760721.429044:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17948672, bytes: 4096 00000080:00200000:1.0:1789760721.429053:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.429062:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.429064:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17948672, 17952768) 0 0 00000080:00200000:1.0:1789760721.429068:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.429085:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4382, 4382] 00020000:00200000:1.0:1789760721.429092:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17948672 chunk: [17948672, 17952768) 17952768, 4096 00020000:00200000:1.0:1789760721.429097:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.429115:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4382, 4382] 00000008:00010000:1.0:1789760721.429131:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.429148:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.429159:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17948672, 17952768) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.429172:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17948672=0x111e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.429228:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.429231:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17952768 00000020:00010000:1.0:1789760721.429239:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.429243:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.429246:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.429250:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.429253:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.429260:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.429266:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.429281:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760721.429743:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 17948672<>17952768 00000080:00200000:1.0:1789760721.533076:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4382, 4382] 00000080:00200000:1.0:1789760721.533094:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.533100:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.533119:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17952768 i_blocks 36871 00000020:00200000:1.0:1789760721.533123:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.533142:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.533178:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17952768, count: 4096 00000080:00200000:1.0:1789760721.533186:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17952768, bytes: 4096 00000080:00200000:1.0:1789760721.533206:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.533215:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.533218:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17952768, 17956864) 0 0 00000080:00200000:1.0:1789760721.533222:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.533239:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4383, 4383] 00020000:00200000:1.0:1789760721.533247:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17952768 chunk: [17952768, 17956864) 17956864, 4096 00020000:00200000:1.0:1789760721.533253:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.533273:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4383, 4383] 00000008:00010000:1.0:1789760721.533294:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.533299:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.533311:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17952768, 17956864) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.533338:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17952768=0x111f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.533561:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.533564:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17956864 00000020:00010000:1.0:1789760721.533574:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.533578:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.533581:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.533584:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.533587:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.533595:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.533602:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.533621:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760721.533723:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 17952768<>17956864 00000080:00200000:1.0:1789760721.624723:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4383, 4383] 00000080:00200000:1.0:1789760721.624742:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.624748:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.624766:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17956864 i_blocks 36871 00000020:00200000:1.0:1789760721.624769:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.624779:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.624817:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17956864, count: 4096 00000080:00200000:1.0:1789760721.624824:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17956864, bytes: 4096 00000080:00200000:1.0:1789760721.624834:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.624843:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.624846:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17956864, 17960960) 0 0 00000080:00200000:1.0:1789760721.624850:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.624869:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4384, 4384] 00020000:00200000:1.0:1789760721.624880:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17956864 chunk: [17956864, 17960960) 17960960, 4096 00020000:00200000:1.0:1789760721.624886:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.624967:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4384, 4384] 00000008:00010000:1.0:1789760721.624989:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.624996:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.625010:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17956864, 17960960) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.625024:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17956864=0x1120000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.625090:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.625094:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17960960 00000020:00010000:1.0:1789760721.625104:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.625108:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.625111:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.625116:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.625119:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.625129:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.625327:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.625350:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760721.625501:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 17956864<>17960960 00000080:00200000:1.0:1789760721.722421:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4384, 4384] 00000080:00200000:1.0:1789760721.722439:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.722445:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.722462:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17960960 i_blocks 36871 00000020:00200000:1.0:1789760721.722466:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.722477:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.722511:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17960960, count: 4096 00000080:00200000:1.0:1789760721.722519:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17960960, bytes: 4096 00000080:00200000:1.0:1789760721.722531:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.722540:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.722543:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17960960, 17965056) 0 0 00000080:00200000:1.0:1789760721.722547:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.722566:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4385, 4385] 00020000:00200000:1.0:1789760721.722575:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17960960 chunk: [17960960, 17965056) 17965056, 4096 00020000:00200000:1.0:1789760721.722580:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.722602:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4385, 4385] 00000008:00010000:1.0:1789760721.722622:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.722627:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.722639:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17960960, 17965056) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.722654:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17960960=0x1121000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.722722:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.722725:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17965056 00000020:00010000:1.0:1789760721.722733:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.722737:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.722740:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.722744:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.722747:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.722754:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.722761:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.722777:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760721.722890:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 17960960<>17965056 00000080:00200000:1.0:1789760721.753941:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4385, 4385] 00000080:00200000:1.0:1789760721.753971:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.753979:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.754000:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17965056 i_blocks 36871 00000020:00200000:1.0:1789760721.754006:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.754025:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.754253:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17965056, count: 4096 00000080:00200000:1.0:1789760721.754264:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17965056, bytes: 4096 00000080:00200000:1.0:1789760721.754281:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.754292:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.754295:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17965056, 17969152) 0 0 00000080:00200000:1.0:1789760721.754300:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.754321:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4386, 4386] 00020000:00200000:1.0:1789760721.754330:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17965056 chunk: [17965056, 17969152) 17969152, 4096 00020000:00200000:1.0:1789760721.754336:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.754363:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4386, 4386] 00000008:00010000:1.0:1789760721.754514:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.754521:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.754534:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17965056, 17969152) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.754552:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17965056=0x1122000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.754626:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.754630:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17969152 00000020:00010000:1.0:1789760721.754638:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.754642:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.754645:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.754648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.754651:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.754661:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.754669:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.754769:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760721.754891:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 17965056<>17969152 00000080:00200000:1.0:1789760721.878682:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4389, 4389] 00000080:00200000:1.0:1789760721.878698:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760721.878703:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760721.878718:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17981440 i_blocks 36871 00000020:00200000:1.0:1789760721.878721:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760721.878730:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760721.878759:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17981440, count: 4096 00000080:00200000:1.0:1789760721.878764:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17981440, bytes: 4096 00000080:00200000:1.0:1789760721.878773:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760721.878780:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760721.878782:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17981440, 17985536) 0 0 00000080:00200000:1.0:1789760721.878786:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760721.878800:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4390, 4390] 00020000:00200000:1.0:1789760721.878807:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17981440 chunk: [17981440, 17985536) 17985536, 4096 00020000:00200000:1.0:1789760721.878811:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760721.878827:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4390, 4390] 00000008:00010000:1.0:1789760721.878843:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760721.878849:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760721.878861:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17981440, 17985536) append 0 append lockpos 0 00000080:00200000:1.0:1789760721.878873:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17981440=0x1126000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760721.878931:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760721.878934:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 17985536 00000020:00010000:1.0:1789760721.878943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760721.878946:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.878949:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760721.878953:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760721.878956:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760721.878972:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760721.878979:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760721.878996:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760721.879106:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 17981440<>17985536 00000100:00100000:1.0:1789760721.879148:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760721.879165:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507190784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760722.002601:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507191168:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760722.058414:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4393, 4393] 00000080:00200000:1.0:1789760722.058442:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.058453:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.058472:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 17997824 i_blocks 36871 00000020:00200000:1.0:1789760722.058476:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.058490:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.058524:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 17997824, count: 4096 00000080:00200000:1.0:1789760722.058530:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 17997824, bytes: 4096 00000080:00200000:1.0:1789760722.058541:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.058550:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.058552:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [17997824, 18001920) 0 0 00000080:00200000:1.0:1789760722.058555:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.058571:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4394, 4394] 00020000:00200000:1.0:1789760722.058582:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 17997824 chunk: [17997824, 18001920) 18001920, 4096 00020000:00200000:1.0:1789760722.058587:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.058987:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4394, 4394] 00000008:00010000:1.0:1789760722.059008:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.059013:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.059025:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [17997824, 18001920) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.059040:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=17997824=0x112a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.059108:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.059111:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18001920 00000020:00010000:1.0:1789760722.059121:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.059124:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.059127:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.059204:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.059207:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.059217:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.059225:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.059246:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760722.059356:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 17997824<>18001920 00000080:00200000:1.0:1789760722.089851:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4394, 4394] 00000080:00200000:1.0:1789760722.089867:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.089873:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.089888:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18001920 i_blocks 36871 00000020:00200000:1.0:1789760722.089892:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.089901:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.089933:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18001920, count: 4096 00000080:00200000:1.0:1789760722.089939:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18001920, bytes: 4096 00000080:00200000:1.0:1789760722.089947:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.089956:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.089958:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18001920, 18006016) 0 0 00000080:00200000:1.0:1789760722.089963:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.089978:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4395, 4395] 00020000:00200000:1.0:1789760722.089986:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18001920 chunk: [18001920, 18006016) 18006016, 4096 00020000:00200000:1.0:1789760722.089990:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.090008:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4395, 4395] 00000008:00010000:1.0:1789760722.090025:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.090030:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.090040:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18001920, 18006016) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.090052:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18001920=0x112b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.090101:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.090104:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18006016 00000020:00010000:1.0:1789760722.090112:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.090115:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.090118:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.090121:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.090124:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.090142:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.090149:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.090174:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760722.090275:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18001920<>18006016 00000080:00200000:1.0:1789760722.120067:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4395, 4395] 00000080:00200000:1.0:1789760722.120083:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.120089:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.120104:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18006016 i_blocks 36871 00000020:00200000:1.0:1789760722.120108:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.120116:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.120161:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18006016, count: 4096 00000080:00200000:1.0:1789760722.120179:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18006016, bytes: 4096 00000080:00200000:1.0:1789760722.120187:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.120196:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.120199:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18006016, 18010112) 0 0 00000080:00200000:1.0:1789760722.120204:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.120220:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4396, 4396] 00020000:00200000:1.0:1789760722.120228:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18006016 chunk: [18006016, 18010112) 18010112, 4096 00020000:00200000:1.0:1789760722.120232:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.120253:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4396, 4396] 00000008:00010000:1.0:1789760722.120270:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.120275:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.120287:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18006016, 18010112) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.120301:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18006016=0x112c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.120358:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.120361:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18010112 00000020:00010000:1.0:1789760722.120370:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.120373:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.120419:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.120424:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.120427:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.120434:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.120441:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.120458:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760722.120568:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18006016<>18010112 00000080:00200000:1.0:1789760722.158663:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4396, 4396] 00000080:00200000:1.0:1789760722.158680:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.158686:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.158702:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18010112 i_blocks 36871 00000020:00200000:1.0:1789760722.158705:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.158716:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.158751:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18010112, count: 4096 00000080:00200000:1.0:1789760722.158758:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18010112, bytes: 4096 00000080:00200000:1.0:1789760722.158768:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.158777:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.158779:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18010112, 18014208) 0 0 00000080:00200000:1.0:1789760722.158783:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.158801:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4397, 4397] 00020000:00200000:1.0:1789760722.158810:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18010112 chunk: [18010112, 18014208) 18014208, 4096 00020000:00200000:1.0:1789760722.158814:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.158834:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4397, 4397] 00000008:00010000:1.0:1789760722.158853:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.158859:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.158871:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18010112, 18014208) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.158885:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18010112=0x112d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.158943:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.158946:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18014208 00000020:00010000:1.0:1789760722.158954:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.158958:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.158961:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.158965:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.158967:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.158975:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.158983:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.158999:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760722.159112:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18010112<>18014208 00000080:00200000:1.0:1789760722.204016:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4397, 4397] 00000080:00200000:1.0:1789760722.204033:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.204039:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.204055:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18014208 i_blocks 36871 00000020:00200000:1.0:1789760722.204058:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.204068:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.204100:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18014208, count: 4096 00000080:00200000:1.0:1789760722.204107:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18014208, bytes: 4096 00000080:00200000:1.0:1789760722.204115:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.204124:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.204126:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18014208, 18018304) 0 0 00000080:00200000:1.0:1789760722.204141:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.204158:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4398, 4398] 00020000:00200000:1.0:1789760722.204166:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18014208 chunk: [18014208, 18018304) 18018304, 4096 00020000:00200000:1.0:1789760722.204171:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.204190:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4398, 4398] 00000008:00010000:1.0:1789760722.204209:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.204215:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.204226:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18014208, 18018304) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.204241:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18014208=0x112e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.204294:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.204297:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18018304 00000020:00010000:1.0:1789760722.204304:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.204307:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.204310:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.204315:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.204318:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.204325:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.204331:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.204347:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760722.204511:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18014208<>18018304 00000080:00200000:1.0:1789760722.287511:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4398, 4398] 00000080:00200000:1.0:1789760722.287528:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.287535:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.287553:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18018304 i_blocks 36871 00000020:00200000:1.0:1789760722.287557:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.287567:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.287602:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18018304, count: 4096 00000080:00200000:1.0:1789760722.287610:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18018304, bytes: 4096 00000080:00200000:1.0:1789760722.287621:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.287631:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.287633:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18018304, 18022400) 0 0 00000080:00200000:1.0:1789760722.287638:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.287656:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4399, 4399] 00020000:00200000:1.0:1789760722.287665:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18018304 chunk: [18018304, 18022400) 18022400, 4096 00020000:00200000:1.0:1789760722.287670:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.287691:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4399, 4399] 00000008:00010000:1.0:1789760722.287711:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.287717:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.287729:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18018304, 18022400) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.287745:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18018304=0x112f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.288618:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.288624:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18022400 00000020:00010000:1.0:1789760722.288635:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.288639:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.288642:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.288646:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.288649:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.288659:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.288667:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.288695:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760722.291240:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18018304<>18022400 00000080:00200000:1.0:1789760722.333884:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4399, 4399] 00000080:00200000:1.0:1789760722.334100:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.334108:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.334124:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18022400 i_blocks 36871 00000020:00200000:1.0:1789760722.334137:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.334146:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.334179:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18022400, count: 4096 00000080:00200000:1.0:1789760722.334186:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18022400, bytes: 4096 00000080:00200000:1.0:1789760722.334195:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.334205:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.334207:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18022400, 18026496) 0 0 00000080:00200000:1.0:1789760722.334211:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.334226:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4400, 4400] 00020000:00200000:1.0:1789760722.334234:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18022400 chunk: [18022400, 18026496) 18026496, 4096 00020000:00200000:1.0:1789760722.334238:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.334256:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4400, 4400] 00000008:00010000:1.0:1789760722.334275:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.334281:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.334292:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18022400, 18026496) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.334306:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18022400=0x1130000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.334359:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.334362:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18026496 00000020:00010000:1.0:1789760722.334371:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.334374:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.334493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.334497:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.334500:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.334509:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.334515:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.334532:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760722.335327:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18022400<>18026496 00000080:00200000:1.0:1789760722.394166:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4400, 4400] 00000080:00200000:1.0:1789760722.394190:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.394197:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.394214:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18026496 i_blocks 36871 00000020:00200000:1.0:1789760722.394219:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.394232:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.394266:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18026496, count: 4096 00000080:00200000:1.0:1789760722.394275:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18026496, bytes: 4096 00000080:00200000:1.0:1789760722.394288:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.394299:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.394301:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18026496, 18030592) 0 0 00000080:00200000:1.0:1789760722.394306:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.394322:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4401, 4401] 00020000:00200000:1.0:1789760722.394331:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18026496 chunk: [18026496, 18030592) 18030592, 4096 00020000:00200000:1.0:1789760722.394336:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.394368:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4401, 4401] 00000008:00010000:1.0:1789760722.394418:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.394426:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.394440:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18026496, 18030592) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.394456:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18026496=0x1131000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.394522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.394525:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18030592 00000020:00010000:1.0:1789760722.394534:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.394537:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.394540:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.394544:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.394547:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.394556:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.394563:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.395234:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760722.395354:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18026496<>18030592 00000080:00200000:1.0:1789760722.450760:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4401, 4401] 00000080:00200000:1.0:1789760722.450776:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.450783:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.450802:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18030592 i_blocks 36871 00000020:00200000:1.0:1789760722.450806:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.450816:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.450850:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18030592, count: 4096 00000080:00200000:1.0:1789760722.450858:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18030592, bytes: 4096 00000080:00200000:1.0:1789760722.450869:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.450879:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.450881:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18030592, 18034688) 0 0 00000080:00200000:1.0:1789760722.450886:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.450956:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4402, 4402] 00020000:00200000:1.0:1789760722.450974:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18030592 chunk: [18030592, 18034688) 18034688, 4096 00020000:00200000:1.0:1789760722.450979:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.451002:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4402, 4402] 00000008:00010000:1.0:1789760722.451020:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.451026:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.451038:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18030592, 18034688) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.451052:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18030592=0x1132000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.451111:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.451114:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18034688 00000020:00010000:1.0:1789760722.451122:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.451140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.451144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.451148:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.451151:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.451159:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.451287:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.451306:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760722.451559:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18030592<>18034688 00000080:00200000:1.0:1789760722.499890:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4402, 4402] 00000080:00200000:1.0:1789760722.499909:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.499915:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.499931:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18034688 i_blocks 36871 00000020:00200000:1.0:1789760722.499935:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.499945:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.500043:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18034688, count: 4096 00000080:00200000:1.0:1789760722.500052:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18034688, bytes: 4096 00000080:00200000:1.0:1789760722.500062:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.500071:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.500074:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18034688, 18038784) 0 0 00000080:00200000:1.0:1789760722.500079:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.500096:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4403, 4403] 00020000:00200000:1.0:1789760722.500104:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18034688 chunk: [18034688, 18038784) 18038784, 4096 00020000:00200000:1.0:1789760722.500109:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.500430:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4403, 4403] 00000008:00010000:1.0:1789760722.500452:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.500459:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.500529:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18034688, 18038784) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.500546:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18034688=0x1133000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.500612:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.500615:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18038784 00000020:00010000:1.0:1789760722.500624:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.500628:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.500631:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.500635:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.500638:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.500647:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.500655:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.500677:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760722.500786:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 18034688<>18038784 00000080:00200000:1.0:1789760722.527640:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4403, 4403] 00000080:00200000:1.0:1789760722.527657:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.527664:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.527681:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18038784 i_blocks 36871 00000020:00200000:1.0:1789760722.527686:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.527695:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.527731:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18038784, count: 4096 00000080:00200000:1.0:1789760722.527738:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18038784, bytes: 4096 00000080:00200000:1.0:1789760722.527746:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.527756:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.527758:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18038784, 18042880) 0 0 00000080:00200000:1.0:1789760722.527762:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.527778:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4404, 4404] 00020000:00200000:1.0:1789760722.527786:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18038784 chunk: [18038784, 18042880) 18042880, 4096 00020000:00200000:1.0:1789760722.527790:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.527810:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4404, 4404] 00000008:00010000:1.0:1789760722.527830:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.527836:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.527848:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18038784, 18042880) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.527862:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18038784=0x1134000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.527921:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.527925:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18042880 00000020:00010000:1.0:1789760722.527933:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.527937:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.527939:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.527943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.527946:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.527954:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.527961:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.527979:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760722.528081:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 18038784<>18042880 00000080:00200000:1.0:1789760722.568961:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4404, 4404] 00000080:00200000:1.0:1789760722.568982:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.568989:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.569006:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18042880 i_blocks 36871 00000020:00200000:1.0:1789760722.569011:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.569021:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.569055:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18042880, count: 4096 00000080:00200000:1.0:1789760722.569062:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18042880, bytes: 4096 00000080:00200000:1.0:1789760722.569072:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.569082:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.569084:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18042880, 18046976) 0 0 00000080:00200000:1.0:1789760722.569089:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.569105:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4405, 4405] 00020000:00200000:1.0:1789760722.569112:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18042880 chunk: [18042880, 18046976) 18046976, 4096 00020000:00200000:1.0:1789760722.569116:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.569146:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4405, 4405] 00000008:00010000:1.0:1789760722.569164:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.569169:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.569181:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18042880, 18046976) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.569195:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18042880=0x1135000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.569250:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.569252:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18046976 00000020:00010000:1.0:1789760722.569260:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.569264:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.569267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.569270:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.569273:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.569280:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.569288:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.569305:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760722.570761:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 18042880<>18046976 00000080:00200000:1.0:1789760722.666264:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4405, 4405] 00000080:00200000:1.0:1789760722.666280:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.666285:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.666300:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18046976 i_blocks 36871 00000020:00200000:1.0:1789760722.666303:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.666312:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.666341:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18046976, count: 4096 00000080:00200000:1.0:1789760722.666347:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18046976, bytes: 4096 00000080:00200000:1.0:1789760722.666354:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.666362:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.666364:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18046976, 18051072) 0 0 00000080:00200000:1.0:1789760722.666368:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.666409:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4406, 4406] 00020000:00200000:1.0:1789760722.666417:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18046976 chunk: [18046976, 18051072) 18051072, 4096 00020000:00200000:1.0:1789760722.666421:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.666439:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4406, 4406] 00000008:00010000:1.0:1789760722.666456:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.666462:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.666471:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18046976, 18051072) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.666482:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18046976=0x1136000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.666529:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.666532:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18051072 00000020:00010000:1.0:1789760722.666541:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.666544:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.666547:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.666550:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.666553:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.666559:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.666565:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.666663:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760722.666759:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 18046976<>18051072 00000080:00200000:1.0:1789760722.718679:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4406, 4406] 00000080:00200000:1.0:1789760722.718692:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.718698:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.718709:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18051072 i_blocks 36871 00000020:00200000:1.0:1789760722.718712:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.718720:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.718747:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18051072, count: 4096 00000080:00200000:1.0:1789760722.718753:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18051072, bytes: 4096 00000080:00200000:1.0:1789760722.718760:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.718768:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.718770:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18051072, 18055168) 0 0 00000080:00200000:1.0:1789760722.718774:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.718788:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4407, 4407] 00020000:00200000:1.0:1789760722.718795:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18051072 chunk: [18051072, 18055168) 18055168, 4096 00020000:00200000:1.0:1789760722.718798:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.718813:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4407, 4407] 00000008:00010000:1.0:1789760722.718830:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.718835:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.718843:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18051072, 18055168) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.718861:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18051072=0x1137000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.718904:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.718906:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18055168 00000020:00010000:1.0:1789760722.718912:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.718915:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.718917:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.718921:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.718923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.718930:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.718935:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.718949:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760722.719019:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18051072<>18055168 00000080:00200000:1.0:1789760722.781659:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4407, 4407] 00000080:00200000:1.0:1789760722.781672:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.781677:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.781688:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18055168 i_blocks 36871 00000020:00200000:1.0:1789760722.781692:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.781701:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.781726:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18055168, count: 4096 00000080:00200000:1.0:1789760722.781733:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18055168, bytes: 4096 00000080:00200000:1.0:1789760722.781740:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.781748:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.781750:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18055168, 18059264) 0 0 00000080:00200000:1.0:1789760722.781753:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.781766:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4408, 4408] 00020000:00200000:1.0:1789760722.781774:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18055168 chunk: [18055168, 18059264) 18059264, 4096 00020000:00200000:1.0:1789760722.781777:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.781791:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4408, 4408] 00000008:00010000:1.0:1789760722.781805:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.781809:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.781818:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18055168, 18059264) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.781828:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18055168=0x1138000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.781870:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.781873:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18059264 00000020:00010000:1.0:1789760722.781879:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.781882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.781885:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.781887:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.781891:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.781897:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.781902:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.781914:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760722.781986:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18055168<>18059264 00000080:00200000:1.0:1789760722.847113:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4408, 4408] 00000080:00200000:1.0:1789760722.847146:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.847152:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.847168:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18059264 i_blocks 36871 00000020:00200000:1.0:1789760722.847173:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.847183:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.847217:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18059264, count: 4096 00000080:00200000:1.0:1789760722.847225:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18059264, bytes: 4096 00000080:00200000:1.0:1789760722.847239:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.847250:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.847252:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18059264, 18063360) 0 0 00000080:00200000:1.0:1789760722.847256:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.847273:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4409, 4409] 00020000:00200000:1.0:1789760722.847281:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18059264 chunk: [18059264, 18063360) 18063360, 4096 00020000:00200000:1.0:1789760722.847285:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.847304:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4409, 4409] 00000008:00010000:1.0:1789760722.847324:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.847329:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.847341:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18059264, 18063360) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.847354:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18059264=0x1139000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.847464:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.847467:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18063360 00000020:00010000:1.0:1789760722.847475:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.847479:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.847482:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.847485:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.847488:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.847495:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.847502:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.847521:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760722.848770:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18059264<>18063360 00000080:00200000:1.0:1789760722.901273:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4409, 4409] 00000080:00200000:1.0:1789760722.901291:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.901298:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.901313:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18063360 i_blocks 36871 00000020:00200000:1.0:1789760722.901317:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.901326:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.901360:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18063360, count: 4096 00000080:00200000:1.0:1789760722.901368:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18063360, bytes: 4096 00000080:00200000:1.0:1789760722.901464:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.901475:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.901477:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18063360, 18067456) 0 0 00000080:00200000:1.0:1789760722.901482:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.901499:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4410, 4410] 00020000:00200000:1.0:1789760722.901508:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18063360 chunk: [18063360, 18067456) 18067456, 4096 00020000:00200000:1.0:1789760722.901512:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.901531:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4410, 4410] 00000008:00010000:1.0:1789760722.901550:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.901556:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.901687:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18063360, 18067456) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.901704:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18063360=0x113a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.901759:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.901762:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18067456 00000020:00010000:1.0:1789760722.901771:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.901774:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.901848:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.901854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.901857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.901866:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.901873:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.901892:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760722.902115:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18063360<>18067456 00000080:00200000:1.0:1789760722.969734:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4410, 4410] 00000080:00200000:1.0:1789760722.969750:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760722.969755:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760722.969770:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18067456 i_blocks 36871 00000020:00200000:1.0:1789760722.969773:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760722.969783:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760722.969816:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18067456, count: 4096 00000080:00200000:1.0:1789760722.969822:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18067456, bytes: 4096 00000080:00200000:1.0:1789760722.969832:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760722.969841:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760722.969842:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18067456, 18071552) 0 0 00000080:00200000:1.0:1789760722.969846:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760722.969860:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4411, 4411] 00020000:00200000:1.0:1789760722.969867:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18067456 chunk: [18067456, 18071552) 18071552, 4096 00020000:00200000:1.0:1789760722.969870:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760722.969886:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4411, 4411] 00000008:00010000:1.0:1789760722.969901:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760722.969906:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760722.969916:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18067456, 18071552) append 0 append lockpos 0 00000080:00200000:1.0:1789760722.969928:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18067456=0x113b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760722.969985:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760722.969988:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18071552 00000020:00010000:1.0:1789760722.970008:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760722.970012:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.970015:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760722.970020:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760722.970023:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760722.970031:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760722.970038:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760722.970056:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760722.970339:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18067456<>18071552 00000080:00200000:1.0:1789760723.010762:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4411, 4411] 00000080:00200000:1.0:1789760723.010780:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.010787:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.010802:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18071552 i_blocks 36871 00000020:00200000:1.0:1789760723.010806:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.010817:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.010856:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18071552, count: 4096 00000080:00200000:1.0:1789760723.010864:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18071552, bytes: 4096 00000080:00200000:1.0:1789760723.010875:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.010993:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.010996:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18071552, 18075648) 0 0 00000080:00200000:1.0:1789760723.011001:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.011022:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4412, 4412] 00020000:00200000:1.0:1789760723.011096:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18071552 chunk: [18071552, 18075648) 18075648, 4096 00020000:00200000:1.0:1789760723.011102:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.011134:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4412, 4412] 00000008:00010000:1.0:1789760723.011154:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.011159:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.011178:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18071552, 18075648) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.011197:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18071552=0x113c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.011259:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.011262:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18075648 00000020:00010000:1.0:1789760723.011271:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.011275:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.011278:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.011281:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.011284:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.011291:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.011298:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.011316:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760723.011632:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18071552<>18075648 00000080:00200000:1.0:1789760723.193848:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4415, 4415] 00000080:00200000:1.0:1789760723.193864:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.193870:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.193884:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18087936 i_blocks 36871 00000020:00200000:1.0:1789760723.193888:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.193898:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.193934:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18087936, count: 4096 00000080:00200000:1.0:1789760723.193942:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18087936, bytes: 4096 00000080:00200000:1.0:1789760723.193953:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.193962:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.193965:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18087936, 18092032) 0 0 00000080:00200000:1.0:1789760723.193969:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.193988:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4416, 4416] 00020000:00200000:1.0:1789760723.193997:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18087936 chunk: [18087936, 18092032) 18092032, 4096 00020000:00200000:1.0:1789760723.194002:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.194023:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4416, 4416] 00000008:00010000:1.0:1789760723.194041:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.194047:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.194059:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18087936, 18092032) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.194073:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18087936=0x1140000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.194139:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.194142:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18092032 00000020:00010000:1.0:1789760723.194151:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.194154:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.194157:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.194160:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.194163:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.194170:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.194177:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.194193:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760723.194306:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18087936<>18092032 00000080:00200000:1.0:1789760723.238511:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4416, 4416] 00000080:00200000:1.0:1789760723.238540:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.238547:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.238566:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18092032 i_blocks 36871 00000020:00200000:1.0:1789760723.238571:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.238588:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.238622:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18092032, count: 4096 00000080:00200000:1.0:1789760723.238631:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18092032, bytes: 4096 00000080:00200000:1.0:1789760723.238642:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.238653:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.238656:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18092032, 18096128) 0 0 00000080:00200000:1.0:1789760723.238661:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.238678:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4417, 4417] 00020000:00200000:1.0:1789760723.238686:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18092032 chunk: [18092032, 18096128) 18096128, 4096 00020000:00200000:1.0:1789760723.238691:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.238828:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4417, 4417] 00000008:00010000:1.0:1789760723.238850:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.238856:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.238868:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18092032, 18096128) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.238882:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18092032=0x1141000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.238945:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.238949:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18096128 00000020:00010000:1.0:1789760723.238957:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.238961:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.238963:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.238967:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.238970:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.238978:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.238985:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.239004:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760723.239278:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18092032<>18096128 00000080:00200000:1.0:1789760723.277602:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4417, 4417] 00000080:00200000:1.0:1789760723.277623:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.277630:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.277648:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18096128 i_blocks 36871 00000020:00200000:1.0:1789760723.277652:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.277664:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.277700:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18096128, count: 4096 00000080:00200000:1.0:1789760723.277708:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18096128, bytes: 4096 00000080:00200000:1.0:1789760723.277719:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.277730:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.277732:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18096128, 18100224) 0 0 00000080:00200000:1.0:1789760723.277737:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.277754:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4418, 4418] 00020000:00200000:1.0:1789760723.277763:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18096128 chunk: [18096128, 18100224) 18100224, 4096 00020000:00200000:1.0:1789760723.277767:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.277787:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4418, 4418] 00000008:00010000:1.0:1789760723.277806:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.277812:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.277824:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18096128, 18100224) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.277841:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18096128=0x1142000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.277905:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.277908:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18100224 00000020:00010000:1.0:1789760723.277917:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.277921:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.277924:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.277928:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.277931:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.277940:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.277947:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.277966:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760723.278093:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18096128<>18100224 00000080:00200000:1.0:1789760723.333848:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4418, 4418] 00000080:00200000:1.0:1789760723.333865:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.333871:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.333886:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18100224 i_blocks 36871 00000020:00200000:1.0:1789760723.333889:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.333911:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.333945:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18100224, count: 4096 00000080:00200000:1.0:1789760723.333953:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18100224, bytes: 4096 00000080:00200000:1.0:1789760723.333963:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.333973:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.333975:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18100224, 18104320) 0 0 00000080:00200000:1.0:1789760723.333980:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.333997:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4419, 4419] 00020000:00200000:1.0:1789760723.334100:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18100224 chunk: [18100224, 18104320) 18104320, 4096 00020000:00200000:1.0:1789760723.334106:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.334137:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4419, 4419] 00000008:00010000:1.0:1789760723.334163:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.334169:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.334180:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18100224, 18104320) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.334195:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18100224=0x1143000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.334254:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.334258:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18104320 00000020:00010000:1.0:1789760723.334266:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.334270:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.334273:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.334277:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.334280:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.334288:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.334295:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.334312:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760723.334472:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18100224<>18104320 00000080:00200000:1.0:1789760723.384766:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4419, 4419] 00000080:00200000:1.0:1789760723.384783:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.384788:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.384804:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18104320 i_blocks 36871 00000020:00200000:1.0:1789760723.384808:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.384817:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.384864:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18104320, count: 4096 00000080:00200000:1.0:1789760723.384872:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18104320, bytes: 4096 00000080:00200000:1.0:1789760723.384883:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.384893:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.385017:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18104320, 18108416) 0 0 00000080:00200000:1.0:1789760723.385024:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.385043:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4420, 4420] 00020000:00200000:1.0:1789760723.385052:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18104320 chunk: [18104320, 18108416) 18108416, 4096 00020000:00200000:1.0:1789760723.385057:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.385078:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4420, 4420] 00000008:00010000:1.0:1789760723.385096:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.385101:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.385113:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18104320, 18108416) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.385138:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18104320=0x1144000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.385207:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.385210:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18108416 00000020:00010000:1.0:1789760723.385218:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.385222:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.385224:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.385228:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.385230:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.385238:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.385245:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.385262:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760723.385371:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18104320<>18108416 00000080:00200000:1.0:1789760723.423039:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4420, 4420] 00000080:00200000:1.0:1789760723.423056:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.423063:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.423079:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18108416 i_blocks 36871 00000020:00200000:1.0:1789760723.423083:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.423094:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.423141:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18108416, count: 4096 00000080:00200000:1.0:1789760723.423149:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18108416, bytes: 4096 00000080:00200000:1.0:1789760723.423169:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.423186:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.423189:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18108416, 18112512) 0 0 00000080:00200000:1.0:1789760723.423194:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.423211:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4421, 4421] 00020000:00200000:1.0:1789760723.423219:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18108416 chunk: [18108416, 18112512) 18112512, 4096 00020000:00200000:1.0:1789760723.423223:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.423246:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4421, 4421] 00000008:00010000:1.0:1789760723.423267:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.423272:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.423282:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18108416, 18112512) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.423294:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18108416=0x1145000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.423343:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.423345:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18112512 00000020:00010000:1.0:1789760723.423353:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.423355:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.423357:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.423360:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.423362:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.423368:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.423375:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.423421:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760723.423530:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18108416<>18112512 00000080:00200000:1.0:1789760723.470743:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4421, 4421] 00000080:00200000:1.0:1789760723.470764:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.470770:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.470787:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18112512 i_blocks 36871 00000020:00200000:1.0:1789760723.470791:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.470802:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.470838:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18112512, count: 4096 00000080:00200000:1.0:1789760723.470845:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18112512, bytes: 4096 00000080:00200000:1.0:1789760723.470857:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.470867:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.470870:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18112512, 18116608) 0 0 00000080:00200000:1.0:1789760723.470874:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.470891:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4422, 4422] 00020000:00200000:1.0:1789760723.470899:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18112512 chunk: [18112512, 18116608) 18116608, 4096 00020000:00200000:1.0:1789760723.470903:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.470925:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4422, 4422] 00000008:00010000:1.0:1789760723.470946:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.470952:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.470965:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18112512, 18116608) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.470980:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18112512=0x1146000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.471041:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.471044:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18116608 00000020:00010000:1.0:1789760723.471052:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.471056:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.471059:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.471063:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.471066:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.471074:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.471082:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.471107:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760723.471411:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 18112512<>18116608 00000080:00200000:1.0:1789760723.500138:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4422, 4422] 00000080:00200000:1.0:1789760723.500157:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.500163:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.500180:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18116608 i_blocks 36871 00000020:00200000:1.0:1789760723.500184:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.500195:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.500230:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18116608, count: 4096 00000080:00200000:1.0:1789760723.500239:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18116608, bytes: 4096 00000080:00200000:1.0:1789760723.500249:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.500259:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.500261:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18116608, 18120704) 0 0 00000080:00200000:1.0:1789760723.500265:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.500283:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4423, 4423] 00020000:00200000:1.0:1789760723.500292:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18116608 chunk: [18116608, 18120704) 18120704, 4096 00020000:00200000:1.0:1789760723.500298:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.500318:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4423, 4423] 00000008:00010000:1.0:1789760723.500339:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.500345:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.500356:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18116608, 18120704) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.500371:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18116608=0x1147000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.500459:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.500462:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18120704 00000020:00010000:1.0:1789760723.500469:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.500471:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.500473:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.500476:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.500478:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.500483:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.500581:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.500598:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760723.500751:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 18116608<>18120704 00000080:00200000:1.0:1789760723.565820:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4423, 4423] 00000080:00200000:1.0:1789760723.565833:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.565839:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.565850:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18120704 i_blocks 36871 00000020:00200000:1.0:1789760723.565854:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.565861:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.565884:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18120704, count: 4096 00000080:00200000:1.0:1789760723.565890:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18120704, bytes: 4096 00000080:00200000:1.0:1789760723.565897:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.565905:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.565907:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18120704, 18124800) 0 0 00000080:00200000:1.0:1789760723.565911:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.565923:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4424, 4424] 00020000:00200000:1.0:1789760723.565930:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18120704 chunk: [18120704, 18124800) 18124800, 4096 00020000:00200000:1.0:1789760723.565933:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.565949:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4424, 4424] 00000008:00010000:1.0:1789760723.565962:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.565966:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.565974:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18120704, 18124800) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.565986:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18120704=0x1148000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.566092:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.566095:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18124800 00000020:00010000:1.0:1789760723.566102:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.566105:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.566107:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.566110:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.566113:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.566301:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.566308:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.566321:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760723.566660:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 18120704<>18124800 00000080:00200000:1.0:1789760723.825870:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4426, 4426] 00000080:00200000:1.0:1789760723.825882:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.825887:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.825898:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18132992 i_blocks 36871 00000020:00200000:1.0:1789760723.825902:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.825910:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.825937:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18132992, count: 4096 00000080:00200000:1.0:1789760723.825943:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18132992, bytes: 4096 00000080:00200000:1.0:1789760723.825954:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.825962:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.825964:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18132992, 18137088) 0 0 00000080:00200000:1.0:1789760723.825968:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.825980:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4427, 4427] 00020000:00200000:1.0:1789760723.825987:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18132992 chunk: [18132992, 18137088) 18137088, 4096 00020000:00200000:1.0:1789760723.825991:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.826006:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4427, 4427] 00000008:00010000:1.0:1789760723.826022:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.826026:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.826036:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18132992, 18137088) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.826047:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18132992=0x114b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.826089:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.826092:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18137088 00000020:00010000:1.0:1789760723.826106:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.826109:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.826112:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.826125:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.826128:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.826135:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.826141:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.826155:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760723.826235:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 18132992<>18137088 00000100:00100000:1.0:1789760723.826269:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760723.826284:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507195520:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760723.879767:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4427, 4427] 00000080:00200000:1.0:1789760723.879779:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.879784:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.879796:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18137088 i_blocks 36871 00000020:00200000:1.0:1789760723.879799:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.879808:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.879837:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18137088, count: 4096 00000080:00200000:1.0:1789760723.879844:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18137088, bytes: 4096 00000080:00200000:1.0:1789760723.879851:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.879859:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.879861:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18137088, 18141184) 0 0 00000080:00200000:1.0:1789760723.879865:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.879879:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4428, 4428] 00020000:00200000:1.0:1789760723.879885:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18137088 chunk: [18137088, 18141184) 18141184, 4096 00020000:00200000:1.0:1789760723.879889:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.879903:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4428, 4428] 00000008:00010000:1.0:1789760723.879917:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.879922:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.879932:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18137088, 18141184) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.879942:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18137088=0x114c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.879983:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.879985:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18141184 00000020:00010000:1.0:1789760723.879993:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.879996:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.879999:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.880001:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.880004:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.880010:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.880014:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.880027:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760723.880175:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18137088<>18141184 00000080:00200000:1.0:1789760723.929593:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4428, 4428] 00000080:00200000:1.0:1789760723.929606:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.929611:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.929622:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18141184 i_blocks 36871 00000020:00200000:1.0:1789760723.929626:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.929634:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.929659:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18141184, count: 4096 00000080:00200000:1.0:1789760723.929665:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18141184, bytes: 4096 00000080:00200000:1.0:1789760723.929826:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.929835:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.929837:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18141184, 18145280) 0 0 00000080:00200000:1.0:1789760723.929841:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.929854:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4429, 4429] 00020000:00200000:1.0:1789760723.929861:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18141184 chunk: [18141184, 18145280) 18145280, 4096 00020000:00200000:1.0:1789760723.929864:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.929879:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4429, 4429] 00000008:00010000:1.0:1789760723.929892:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.929896:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.929905:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18141184, 18145280) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.929915:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18141184=0x114d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.929956:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.929958:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18145280 00000020:00010000:1.0:1789760723.929966:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.929969:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.929971:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.929974:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.929977:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.929984:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.929989:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.930002:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760723.930076:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18141184<>18145280 00000080:00200000:1.0:1789760723.975245:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4429, 4429] 00000080:00200000:1.0:1789760723.975259:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760723.975264:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760723.975276:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18145280 i_blocks 36871 00000020:00200000:1.0:1789760723.975279:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760723.975288:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760723.975315:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18145280, count: 4096 00000080:00200000:1.0:1789760723.975321:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18145280, bytes: 4096 00000080:00200000:1.0:1789760723.975329:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760723.975336:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760723.975338:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18145280, 18149376) 0 0 00000080:00200000:1.0:1789760723.975342:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760723.975354:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4430, 4430] 00020000:00200000:1.0:1789760723.975360:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18145280 chunk: [18145280, 18149376) 18149376, 4096 00020000:00200000:1.0:1789760723.975631:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760723.975647:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4430, 4430] 00000008:00010000:1.0:1789760723.975661:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760723.975666:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760723.976071:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18145280, 18149376) append 0 append lockpos 0 00000080:00200000:1.0:1789760723.976083:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18145280=0x114e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760723.976143:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760723.976147:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18149376 00000020:00010000:1.0:1789760723.976215:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760723.976219:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.976222:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760723.976226:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760723.976236:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760723.976243:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760723.976249:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760723.976266:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760723.976338:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18145280<>18149376 00000080:00200000:1.0:1789760724.013505:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4430, 4430] 00000080:00200000:1.0:1789760724.013518:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.013523:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.013535:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18149376 i_blocks 36871 00000020:00200000:1.0:1789760724.013538:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.013546:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.013583:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18149376, count: 4096 00000080:00200000:1.0:1789760724.013590:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18149376, bytes: 4096 00000080:00200000:1.0:1789760724.013598:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.013605:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.013607:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18149376, 18153472) 0 0 00000080:00200000:1.0:1789760724.013611:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.013624:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4431, 4431] 00020000:00200000:1.0:1789760724.013631:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18149376 chunk: [18149376, 18153472) 18153472, 4096 00020000:00200000:1.0:1789760724.013634:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.013649:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4431, 4431] 00000008:00010000:1.0:1789760724.013662:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.013667:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.013676:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18149376, 18153472) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.013688:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18149376=0x114f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.013731:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.013733:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18153472 00000020:00010000:1.0:1789760724.013740:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.013743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.013745:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.013748:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.013750:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.013757:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.013763:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.013776:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760724.013849:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18149376<>18153472 00000080:00200000:1.0:1789760724.108180:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4431, 4431] 00000080:00200000:1.0:1789760724.108193:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.108198:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.108210:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18153472 i_blocks 36871 00000020:00200000:1.0:1789760724.108217:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.108225:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.108250:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18153472, count: 4096 00000080:00200000:1.0:1789760724.108256:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18153472, bytes: 4096 00000080:00200000:1.0:1789760724.108263:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.108270:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.108272:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18153472, 18157568) 0 0 00000080:00200000:1.0:1789760724.108276:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.108293:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4432, 4432] 00020000:00200000:1.0:1789760724.108299:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18153472 chunk: [18153472, 18157568) 18157568, 4096 00020000:00200000:1.0:1789760724.108305:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.108318:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4432, 4432] 00000008:00010000:1.0:1789760724.108333:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.108337:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.108344:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18153472, 18157568) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.108353:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18153472=0x1150000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.108408:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.108411:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18157568 00000020:00010000:1.0:1789760724.108417:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.108420:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.108423:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.108426:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.108428:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.108435:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.108441:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.108454:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760724.108541:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18153472<>18157568 00000080:00200000:1.0:1789760724.180809:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4432, 4432] 00000080:00200000:1.0:1789760724.180822:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.180827:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.180839:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18157568 i_blocks 36871 00000020:00200000:1.0:1789760724.180842:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.180849:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.180876:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18157568, count: 4096 00000080:00200000:1.0:1789760724.180881:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18157568, bytes: 4096 00000080:00200000:1.0:1789760724.180889:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.180895:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.180898:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18157568, 18161664) 0 0 00000080:00200000:1.0:1789760724.180901:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.180912:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4433, 4433] 00020000:00200000:1.0:1789760724.180919:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18157568 chunk: [18157568, 18161664) 18161664, 4096 00020000:00200000:1.0:1789760724.180923:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.180937:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4433, 4433] 00000008:00010000:1.0:1789760724.180950:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.180954:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.180964:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18157568, 18161664) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.180974:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18157568=0x1151000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.181017:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.181020:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18161664 00000020:00010000:1.0:1789760724.181026:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.181029:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.181032:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.181035:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.181038:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.181044:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.181049:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.181063:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760724.181217:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18157568<>18161664 00000080:00200000:1.0:1789760724.240601:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4433, 4433] 00000080:00200000:1.0:1789760724.240614:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.240619:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.240632:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18161664 i_blocks 36871 00000020:00200000:1.0:1789760724.240635:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.240642:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.240669:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18161664, count: 4096 00000080:00200000:1.0:1789760724.240676:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18161664, bytes: 4096 00000080:00200000:1.0:1789760724.240683:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.240690:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.240692:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18161664, 18165760) 0 0 00000080:00200000:1.0:1789760724.240696:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.240707:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4434, 4434] 00020000:00200000:1.0:1789760724.240714:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18161664 chunk: [18161664, 18165760) 18165760, 4096 00020000:00200000:1.0:1789760724.240718:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.240732:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4434, 4434] 00000008:00010000:1.0:1789760724.240746:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.240751:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.240759:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18161664, 18165760) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.240769:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18161664=0x1152000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.240810:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.240813:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18165760 00000020:00010000:1.0:1789760724.240820:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.240823:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.240825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.240828:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.240831:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.240837:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.240842:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.240856:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760724.240934:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18161664<>18165760 00000080:00200000:1.0:1789760724.284757:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4434, 4434] 00000080:00200000:1.0:1789760724.284771:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.284776:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.284788:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18165760 i_blocks 36871 00000020:00200000:1.0:1789760724.284791:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.284799:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.284827:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18165760, count: 4096 00000080:00200000:1.0:1789760724.284833:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18165760, bytes: 4096 00000080:00200000:1.0:1789760724.284841:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.284848:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.284850:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18165760, 18169856) 0 0 00000080:00200000:1.0:1789760724.284854:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.284867:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4435, 4435] 00020000:00200000:1.0:1789760724.284873:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18165760 chunk: [18165760, 18169856) 18169856, 4096 00020000:00200000:1.0:1789760724.284876:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.284890:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4435, 4435] 00000008:00010000:1.0:1789760724.284913:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.284918:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.284927:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18165760, 18169856) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.284937:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18165760=0x1153000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.284981:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.284983:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18169856 00000020:00010000:1.0:1789760724.284991:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.284995:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.284998:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.285001:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.285003:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.285009:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.285015:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.285028:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760724.285098:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18165760<>18169856 00000080:00200000:1.0:1789760724.329467:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4435, 4435] 00000080:00200000:1.0:1789760724.329481:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.329487:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.329500:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18169856 i_blocks 36871 00000020:00200000:1.0:1789760724.329504:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.329513:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.329538:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18169856, count: 4096 00000080:00200000:1.0:1789760724.329545:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18169856, bytes: 4096 00000080:00200000:1.0:1789760724.329553:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.329572:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.329574:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18169856, 18173952) 0 0 00000080:00200000:1.0:1789760724.329578:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.329590:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4436, 4436] 00020000:00200000:1.0:1789760724.329596:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18169856 chunk: [18169856, 18173952) 18173952, 4096 00020000:00200000:1.0:1789760724.329600:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.329614:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4436, 4436] 00000008:00010000:1.0:1789760724.329627:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.329631:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.329639:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18169856, 18173952) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.329649:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18169856=0x1154000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.329699:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.329702:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18173952 00000020:00010000:1.0:1789760724.329708:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.329711:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.329714:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.329717:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.329719:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.329725:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.329731:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.329745:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760724.329821:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 18169856<>18173952 00000080:00200000:1.0:1789760724.372786:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4436, 4436] 00000080:00200000:1.0:1789760724.372800:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.372805:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.372817:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18173952 i_blocks 36871 00000020:00200000:1.0:1789760724.372821:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.372840:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.372866:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18173952, count: 4096 00000080:00200000:1.0:1789760724.372873:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18173952, bytes: 4096 00000080:00200000:1.0:1789760724.372880:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.372888:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.372891:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18173952, 18178048) 0 0 00000080:00200000:1.0:1789760724.372895:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.372908:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4437, 4437] 00020000:00200000:1.0:1789760724.372916:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18173952 chunk: [18173952, 18178048) 18178048, 4096 00020000:00200000:1.0:1789760724.372921:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.372936:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4437, 4437] 00000008:00010000:1.0:1789760724.372950:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.372954:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.372964:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18173952, 18178048) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.372975:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18173952=0x1155000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.373024:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.373026:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18178048 00000020:00010000:1.0:1789760724.373034:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.373037:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.373040:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.373044:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.373047:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.373054:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.373060:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.373074:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760724.373168:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 18173952<>18178048 00000080:00200000:1.0:1789760724.445203:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4437, 4437] 00000080:00200000:1.0:1789760724.445263:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.445269:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.445281:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18178048 i_blocks 36871 00000020:00200000:1.0:1789760724.445285:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.445293:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.445320:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18178048, count: 4096 00000080:00200000:1.0:1789760724.445327:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18178048, bytes: 4096 00000080:00200000:1.0:1789760724.445334:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.445342:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.445344:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18178048, 18182144) 0 0 00000080:00200000:1.0:1789760724.445348:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.445361:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4438, 4438] 00020000:00200000:1.0:1789760724.445368:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18178048 chunk: [18178048, 18182144) 18182144, 4096 00020000:00200000:1.0:1789760724.445371:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.445407:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4438, 4438] 00000008:00010000:1.0:1789760724.445424:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.445429:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.445439:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18178048, 18182144) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.445450:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18178048=0x1156000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.445498:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.445501:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18182144 00000020:00010000:1.0:1789760724.445508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.445522:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.445525:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.445528:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.445532:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.445539:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.445545:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.445559:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760724.445652:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 18178048<>18182144 00000080:00200000:1.0:1789760724.479102:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4438, 4438] 00000080:00200000:1.0:1789760724.479125:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.479130:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.479140:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18182144 i_blocks 36871 00000020:00200000:1.0:1789760724.479148:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.479156:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.479185:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18182144, count: 4096 00000080:00200000:1.0:1789760724.479190:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18182144, bytes: 4096 00000080:00200000:1.0:1789760724.479197:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.479202:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.479203:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18182144, 18186240) 0 0 00000080:00200000:1.0:1789760724.479206:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.479215:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4439, 4439] 00020000:00200000:1.0:1789760724.479220:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18182144 chunk: [18182144, 18186240) 18186240, 4096 00020000:00200000:1.0:1789760724.479224:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.479236:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4439, 4439] 00000008:00010000:1.0:1789760724.479247:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.479251:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.479258:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18182144, 18186240) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.479267:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18182144=0x1157000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.479299:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.479301:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18186240 00000020:00010000:1.0:1789760724.479307:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.479309:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.479311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.479314:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.479315:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.479320:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.479324:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.479334:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760724.479443:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 18182144<>18186240 00000080:00200000:1.0:1789760724.547850:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4439, 4439] 00000080:00200000:1.0:1789760724.547863:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760724.547868:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760724.547879:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18186240 i_blocks 36871 00000020:00200000:1.0:1789760724.547882:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760724.547890:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760724.547915:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18186240, count: 4096 00000080:00200000:1.0:1789760724.547921:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18186240, bytes: 4096 00000080:00200000:1.0:1789760724.547930:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760724.547936:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760724.547938:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18186240, 18190336) 0 0 00000080:00200000:1.0:1789760724.547942:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760724.547953:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4440, 4440] 00020000:00200000:1.0:1789760724.547960:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18186240 chunk: [18186240, 18190336) 18190336, 4096 00020000:00200000:1.0:1789760724.547963:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760724.547979:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4440, 4440] 00000008:00010000:1.0:1789760724.547992:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760724.547996:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760724.548006:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18186240, 18190336) append 0 append lockpos 0 00000080:00200000:1.0:1789760724.548017:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18186240=0x1158000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760724.548058:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760724.548060:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18190336 00000020:00010000:1.0:1789760724.548066:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760724.548069:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.548072:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760724.548075:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760724.548077:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760724.548083:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760724.549418:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760724.549560:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760724.549656:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18186240<>18190336 00000100:00100000:1.0:1789760724.830977:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507198080:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760725.522049:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507199872:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760725.579010:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4461, 4461] 00000080:00200000:1.0:1789760725.579027:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760725.579036:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760725.579052:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18276352 i_blocks 36871 00000020:00200000:1.0:1789760725.579055:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760725.579066:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760725.579111:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18276352, count: 4096 00000080:00200000:1.0:1789760725.579119:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18276352, bytes: 4096 00000080:00200000:1.0:1789760725.579131:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760725.579140:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760725.579143:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18276352, 18280448) 0 0 00000080:00200000:1.0:1789760725.579147:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760725.579163:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4462, 4462] 00020000:00200000:1.0:1789760725.579171:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18276352 chunk: [18276352, 18280448) 18280448, 4096 00020000:00200000:1.0:1789760725.579176:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760725.579196:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4462, 4462] 00000008:00010000:1.0:1789760725.579214:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760725.579221:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760725.579233:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18276352, 18280448) append 0 append lockpos 0 00000080:00200000:1.0:1789760725.579248:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18276352=0x116e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760725.579309:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760725.579312:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18280448 00000020:00010000:1.0:1789760725.579321:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760725.579324:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.579327:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.579331:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760725.579334:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760725.579341:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760725.579348:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760725.579364:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760725.579511:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18276352<>18280448 00000080:00200000:1.0:1789760725.655856:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4462, 4462] 00000080:00200000:1.0:1789760725.655881:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760725.655888:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760725.655908:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18280448 i_blocks 36871 00000020:00200000:1.0:1789760725.655913:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760725.655927:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760725.655964:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18280448, count: 4096 00000080:00200000:1.0:1789760725.655972:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18280448, bytes: 4096 00000080:00200000:1.0:1789760725.655985:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760725.655995:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760725.655997:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18280448, 18284544) 0 0 00000080:00200000:1.0:1789760725.656003:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760725.656020:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4463, 4463] 00020000:00200000:1.0:1789760725.656030:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18280448 chunk: [18280448, 18284544) 18284544, 4096 00020000:00200000:1.0:1789760725.656035:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760725.656057:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4463, 4463] 00000008:00010000:1.0:1789760725.656075:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760725.656081:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760725.656093:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18280448, 18284544) append 0 append lockpos 0 00000080:00200000:1.0:1789760725.656121:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18280448=0x116f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760725.656191:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760725.656194:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18284544 00000020:00010000:1.0:1789760725.656203:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760725.656207:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.656210:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.656214:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760725.656217:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760725.656225:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760725.656232:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760725.656252:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760725.656367:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18280448<>18284544 00000080:00200000:1.0:1789760725.748278:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4463, 4463] 00000080:00200000:1.0:1789760725.748295:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760725.748301:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760725.748317:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18284544 i_blocks 36871 00000020:00200000:1.0:1789760725.748321:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760725.748331:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760725.748365:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18284544, count: 4096 00000080:00200000:1.0:1789760725.748372:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18284544, bytes: 4096 00000080:00200000:1.0:1789760725.748429:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760725.748439:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760725.748441:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18284544, 18288640) 0 0 00000080:00200000:1.0:1789760725.748454:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760725.748472:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4464, 4464] 00020000:00200000:1.0:1789760725.748481:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18284544 chunk: [18284544, 18288640) 18288640, 4096 00020000:00200000:1.0:1789760725.748485:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760725.748507:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4464, 4464] 00000008:00010000:1.0:1789760725.748527:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760725.748533:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760725.748546:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18284544, 18288640) append 0 append lockpos 0 00000080:00200000:1.0:1789760725.748560:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18284544=0x1170000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760725.748615:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760725.748618:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18288640 00000020:00010000:1.0:1789760725.748627:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760725.748630:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.748633:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.748637:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760725.748640:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760725.748649:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760725.748656:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760725.748674:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760725.748787:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18284544<>18288640 00000080:00200000:1.0:1789760725.827613:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4464, 4464] 00000080:00200000:1.0:1789760725.827632:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760725.827638:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760725.827652:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18288640 i_blocks 36871 00000020:00200000:1.0:1789760725.827655:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760725.827665:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760725.827700:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18288640, count: 4096 00000080:00200000:1.0:1789760725.827707:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18288640, bytes: 4096 00000080:00200000:1.0:1789760725.827716:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760725.827724:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760725.827726:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18288640, 18292736) 0 0 00000080:00200000:1.0:1789760725.827729:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760725.827745:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4465, 4465] 00020000:00200000:1.0:1789760725.827752:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18288640 chunk: [18288640, 18292736) 18292736, 4096 00020000:00200000:1.0:1789760725.827952:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760725.827974:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4465, 4465] 00000008:00010000:1.0:1789760725.827993:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760725.827999:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760725.828010:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18288640, 18292736) append 0 append lockpos 0 00000080:00200000:1.0:1789760725.828023:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18288640=0x1171000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760725.828078:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760725.828081:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18292736 00000020:00010000:1.0:1789760725.828088:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760725.828091:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.828093:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.828096:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760725.828099:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760725.828116:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760725.828122:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760725.828144:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760725.828245:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18288640<>18292736 00000080:00200000:1.0:1789760725.876972:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4465, 4465] 00000080:00200000:1.0:1789760725.876989:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760725.876996:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760725.877012:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18292736 i_blocks 36871 00000020:00200000:1.0:1789760725.877015:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760725.877024:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760725.877057:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18292736, count: 4096 00000080:00200000:1.0:1789760725.877064:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18292736, bytes: 4096 00000080:00200000:1.0:1789760725.877074:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760725.877083:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760725.877085:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18292736, 18296832) 0 0 00000080:00200000:1.0:1789760725.877089:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760725.877118:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4466, 4466] 00020000:00200000:1.0:1789760725.877126:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18292736 chunk: [18292736, 18296832) 18296832, 4096 00020000:00200000:1.0:1789760725.877130:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760725.877153:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4466, 4466] 00000008:00010000:1.0:1789760725.877171:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760725.877177:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760725.877189:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18292736, 18296832) append 0 append lockpos 0 00000080:00200000:1.0:1789760725.877202:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18292736=0x1172000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760725.877256:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760725.877259:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18296832 00000020:00010000:1.0:1789760725.877268:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760725.877272:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.877275:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760725.877278:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760725.877281:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760725.877289:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760725.877300:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760725.877317:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760725.877544:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18292736<>18296832 00000100:00100000:1.0:1789760726.034871:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507201024:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760726.065235:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4470, 4470] 00000080:00200000:1.0:1789760726.065254:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.065260:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.065276:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18313216 i_blocks 36871 00000020:00200000:1.0:1789760726.065280:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.065291:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.065322:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18313216, count: 4096 00000080:00200000:1.0:1789760726.065329:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18313216, bytes: 4096 00000080:00200000:1.0:1789760726.065340:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.065348:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.065350:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18313216, 18317312) 0 0 00000080:00200000:1.0:1789760726.065356:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.065373:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4471, 4471] 00020000:00200000:1.0:1789760726.065417:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18313216 chunk: [18313216, 18317312) 18317312, 4096 00020000:00200000:1.0:1789760726.065423:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.065444:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4471, 4471] 00000008:00010000:1.0:1789760726.065462:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.065468:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.065480:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18313216, 18317312) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.065496:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18313216=0x1177000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.065619:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.065622:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18317312 00000020:00010000:1.0:1789760726.065631:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.065634:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.065638:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.065641:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.065644:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.065652:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.065659:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.065676:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760726.065779:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18313216<>18317312 00000080:00200000:1.0:1789760726.096472:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4471, 4471] 00000080:00200000:1.0:1789760726.096490:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.096497:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.096515:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18317312 i_blocks 36871 00000020:00200000:1.0:1789760726.096520:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.096532:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.096581:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18317312, count: 4096 00000080:00200000:1.0:1789760726.096589:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18317312, bytes: 4096 00000080:00200000:1.0:1789760726.096602:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.096613:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.096616:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18317312, 18321408) 0 0 00000080:00200000:1.0:1789760726.096621:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.096640:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4472, 4472] 00020000:00200000:1.0:1789760726.096648:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18317312 chunk: [18317312, 18321408) 18321408, 4096 00020000:00200000:1.0:1789760726.096654:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.096678:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4472, 4472] 00000008:00010000:1.0:1789760726.096701:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.096707:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.096719:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18317312, 18321408) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.096733:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18317312=0x1178000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.096800:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.096803:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18321408 00000020:00010000:1.0:1789760726.096813:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.096817:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.096820:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.096825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.096828:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.096839:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.096847:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.096869:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760726.096986:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18317312<>18321408 00000080:00200000:1.0:1789760726.130511:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4472, 4472] 00000080:00200000:1.0:1789760726.130528:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.130534:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.130560:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18321408 i_blocks 36871 00000020:00200000:1.0:1789760726.130564:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.130574:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.130608:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18321408, count: 4096 00000080:00200000:1.0:1789760726.130615:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18321408, bytes: 4096 00000080:00200000:1.0:1789760726.130625:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.130635:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.130637:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18321408, 18325504) 0 0 00000080:00200000:1.0:1789760726.130643:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.130659:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4473, 4473] 00020000:00200000:1.0:1789760726.130668:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18321408 chunk: [18321408, 18325504) 18325504, 4096 00020000:00200000:1.0:1789760726.130672:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.130692:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4473, 4473] 00000008:00010000:1.0:1789760726.130711:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.130718:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.130730:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18321408, 18325504) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.130745:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18321408=0x1179000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.130804:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.130807:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18325504 00000020:00010000:1.0:1789760726.130816:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.130819:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.130822:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.130825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.130828:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.130836:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.130842:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.130859:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760726.130965:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18321408<>18325504 00000080:00200000:1.0:1789760726.162875:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4473, 4473] 00000080:00200000:1.0:1789760726.162892:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.162899:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.162916:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18325504 i_blocks 36871 00000020:00200000:1.0:1789760726.162921:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.162933:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.162970:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18325504, count: 4096 00000080:00200000:1.0:1789760726.162977:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18325504, bytes: 4096 00000080:00200000:1.0:1789760726.162989:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.162999:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.163002:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18325504, 18329600) 0 0 00000080:00200000:1.0:1789760726.163007:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.163026:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4474, 4474] 00020000:00200000:1.0:1789760726.163035:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18325504 chunk: [18325504, 18329600) 18329600, 4096 00020000:00200000:1.0:1789760726.163040:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.163073:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4474, 4474] 00000008:00010000:1.0:1789760726.163093:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.163109:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.163121:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18325504, 18329600) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.163135:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18325504=0x117a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.163206:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.163209:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18329600 00000020:00010000:1.0:1789760726.163218:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.163222:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.163225:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.163228:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.163232:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.163241:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.163248:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.163264:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760726.163423:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18325504<>18329600 00000080:00200000:1.0:1789760726.366312:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4479, 4479] 00000080:00200000:1.0:1789760726.366330:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.366336:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.366351:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18350080 i_blocks 36871 00000020:00200000:1.0:1789760726.366354:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.366366:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.366423:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18350080, count: 4096 00000080:00200000:1.0:1789760726.366431:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18350080, bytes: 4096 00000080:00200000:1.0:1789760726.366442:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.366452:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.366455:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18350080, 18354176) 0 0 00000080:00200000:1.0:1789760726.366460:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.366479:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4480, 4480] 00020000:00200000:1.0:1789760726.366488:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18350080 chunk: [18350080, 18354176) 18354176, 4096 00020000:00200000:1.0:1789760726.366492:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.366514:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4480, 4480] 00000008:00010000:1.0:1789760726.366534:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.366541:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.366629:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18350080, 18354176) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.366646:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18350080=0x1180000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.366713:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.366716:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18354176 00000020:00010000:1.0:1789760726.366724:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.366728:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.366731:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.366734:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.366737:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.366746:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.366753:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.366773:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760726.366886:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18350080<>18354176 00000080:00200000:1.0:1789760726.399638:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4480, 4480] 00000080:00200000:1.0:1789760726.399657:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.399664:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.399681:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18354176 i_blocks 36871 00000020:00200000:1.0:1789760726.399685:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.399696:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.399731:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18354176, count: 4096 00000080:00200000:1.0:1789760726.399739:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18354176, bytes: 4096 00000080:00200000:1.0:1789760726.399750:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.399759:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.399761:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18354176, 18358272) 0 0 00000080:00200000:1.0:1789760726.399766:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.399785:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4481, 4481] 00020000:00200000:1.0:1789760726.399795:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18354176 chunk: [18354176, 18358272) 18358272, 4096 00020000:00200000:1.0:1789760726.399800:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.399823:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4481, 4481] 00000008:00010000:1.0:1789760726.399845:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.399852:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.399864:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18354176, 18358272) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.399879:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18354176=0x1181000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.399947:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.399950:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18358272 00000020:00010000:1.0:1789760726.399960:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.399963:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.399966:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.399970:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.399973:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.399981:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.399988:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.434560:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4481, 4481] 00000080:00200000:1.0:1789760726.434578:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.434585:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.434602:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18358272 i_blocks 36871 00000020:00200000:1.0:1789760726.434606:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.434616:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.434650:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18358272, count: 4096 00000080:00200000:1.0:1789760726.434658:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18358272, bytes: 4096 00000080:00200000:1.0:1789760726.434667:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.434676:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.434678:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18358272, 18362368) 0 0 00000080:00200000:1.0:1789760726.434682:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.434699:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4482, 4482] 00020000:00200000:1.0:1789760726.434708:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18358272 chunk: [18358272, 18362368) 18362368, 4096 00020000:00200000:1.0:1789760726.434713:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.434735:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4482, 4482] 00000008:00010000:1.0:1789760726.434755:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.434760:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.434772:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18358272, 18362368) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.434787:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18358272=0x1182000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.434851:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.434854:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18362368 00000020:00010000:1.0:1789760726.434863:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.434867:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.434870:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.434874:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.434876:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.434885:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.434892:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.434921:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760726.435029:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18358272<>18362368 00000100:00100000:1.0:1789760726.435073:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760726.435090:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507202560:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760726.464883:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4482, 4482] 00000080:00200000:1.0:1789760726.464909:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.464916:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.464933:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18362368 i_blocks 36871 00000020:00200000:1.0:1789760726.464937:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.464948:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.464984:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18362368, count: 4096 00000080:00200000:1.0:1789760726.464993:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18362368, bytes: 4096 00000080:00200000:1.0:1789760726.465004:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.465013:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.465015:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18362368, 18366464) 0 0 00000080:00200000:1.0:1789760726.465020:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.465039:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4483, 4483] 00020000:00200000:1.0:1789760726.465049:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18362368 chunk: [18362368, 18366464) 18366464, 4096 00020000:00200000:1.0:1789760726.465053:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.465074:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4483, 4483] 00000008:00010000:1.0:1789760726.465092:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.465107:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.465119:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18362368, 18366464) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.465133:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18362368=0x1183000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.465198:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.465201:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18366464 00000020:00010000:1.0:1789760726.465210:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.465213:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.465216:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.465220:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.465223:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.465230:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.465236:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.465253:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760726.465361:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 18362368<>18366464 00000080:00200000:1.0:1789760726.513565:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4483, 4483] 00000080:00200000:1.0:1789760726.513584:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.513592:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.513608:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18366464 i_blocks 36871 00000020:00200000:1.0:1789760726.513613:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.513623:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.513658:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18366464, count: 4096 00000080:00200000:1.0:1789760726.513667:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18366464, bytes: 4096 00000080:00200000:1.0:1789760726.513677:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.513687:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.513690:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18366464, 18370560) 0 0 00000080:00200000:1.0:1789760726.513695:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.513713:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4484, 4484] 00020000:00200000:1.0:1789760726.513722:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18366464 chunk: [18366464, 18370560) 18370560, 4096 00020000:00200000:1.0:1789760726.513727:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.513747:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4484, 4484] 00000008:00010000:1.0:1789760726.513767:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.513774:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.513787:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18366464, 18370560) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.513801:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18366464=0x1184000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.513863:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.513866:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18370560 00000020:00010000:1.0:1789760726.513875:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.513879:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.513882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.513885:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.513889:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.513897:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.513904:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.513921:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760726.514031:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 18366464<>18370560 00000080:00200000:1.0:1789760726.570976:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4484, 4484] 00000080:00200000:1.0:1789760726.570993:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.570999:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.571015:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18370560 i_blocks 36871 00000020:00200000:1.0:1789760726.571019:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.571030:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.571067:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18370560, count: 4096 00000080:00200000:1.0:1789760726.571074:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18370560, bytes: 4096 00000080:00200000:1.0:1789760726.571084:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.571093:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.571105:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18370560, 18374656) 0 0 00000080:00200000:1.0:1789760726.571110:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.571127:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4485, 4485] 00020000:00200000:1.0:1789760726.571135:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18370560 chunk: [18370560, 18374656) 18374656, 4096 00020000:00200000:1.0:1789760726.571140:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.571161:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4485, 4485] 00000008:00010000:1.0:1789760726.571180:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.571186:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.571198:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18370560, 18374656) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.571212:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18370560=0x1185000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.571274:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.571277:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18374656 00000020:00010000:1.0:1789760726.571286:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.571290:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.571293:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.571297:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.571300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.571308:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.571315:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.571333:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760726.571483:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 18370560<>18374656 00000080:00200000:1.0:1789760726.628634:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4485, 4485] 00000080:00200000:1.0:1789760726.628651:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.628658:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.628685:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18374656 i_blocks 36871 00000020:00200000:1.0:1789760726.628690:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.628699:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.628735:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18374656, count: 4096 00000080:00200000:1.0:1789760726.628743:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18374656, bytes: 4096 00000080:00200000:1.0:1789760726.628755:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.628765:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.628767:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18374656, 18378752) 0 0 00000080:00200000:1.0:1789760726.628772:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.628790:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4486, 4486] 00020000:00200000:1.0:1789760726.628798:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18374656 chunk: [18374656, 18378752) 18378752, 4096 00020000:00200000:1.0:1789760726.628802:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.628824:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4486, 4486] 00000008:00010000:1.0:1789760726.628846:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.628852:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.628864:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18374656, 18378752) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.628878:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18374656=0x1186000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.628955:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.628958:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18378752 00000020:00010000:1.0:1789760726.628967:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.628970:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.628973:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.628976:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.628979:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.628987:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.628994:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.629028:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760726.629140:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 18374656<>18378752 00000100:00100000:1.0:1789760726.629206:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760726.629233:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507203072:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760726.671874:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4486, 4486] 00000080:00200000:1.0:1789760726.671892:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.671899:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.671915:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18378752 i_blocks 36871 00000020:00200000:1.0:1789760726.671919:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.671929:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.671969:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18378752, count: 4096 00000080:00200000:1.0:1789760726.671977:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18378752, bytes: 4096 00000080:00200000:1.0:1789760726.671987:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.671997:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.671999:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18378752, 18382848) 0 0 00000080:00200000:1.0:1789760726.672004:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.672103:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4487, 4487] 00020000:00200000:1.0:1789760726.672113:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18378752 chunk: [18378752, 18382848) 18382848, 4096 00020000:00200000:1.0:1789760726.672118:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.672140:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4487, 4487] 00000008:00010000:1.0:1789760726.672159:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.672165:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.672177:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18378752, 18382848) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.672193:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18378752=0x1187000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.672255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.672258:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18382848 00000020:00010000:1.0:1789760726.672267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.672271:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.672273:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.672277:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.672281:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.672290:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.672297:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.672317:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760726.672457:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18378752<>18382848 00000080:00200000:1.0:1789760726.710931:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4487, 4487] 00000080:00200000:1.0:1789760726.710949:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.710956:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.710971:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18382848 i_blocks 36871 00000020:00200000:1.0:1789760726.710976:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.710985:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.711090:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18382848, count: 4096 00000080:00200000:1.0:1789760726.711107:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18382848, bytes: 4096 00000080:00200000:1.0:1789760726.711118:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.711128:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.711130:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18382848, 18386944) 0 0 00000080:00200000:1.0:1789760726.711135:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.711153:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4488, 4488] 00020000:00200000:1.0:1789760726.711161:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18382848 chunk: [18382848, 18386944) 18386944, 4096 00020000:00200000:1.0:1789760726.711165:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.711185:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4488, 4488] 00000008:00010000:1.0:1789760726.711213:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.711220:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.711231:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18382848, 18386944) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.711246:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18382848=0x1188000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.711306:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.711309:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18386944 00000020:00010000:1.0:1789760726.711317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.711321:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.711325:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.711328:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.711332:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.711341:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.711348:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.711366:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760726.711516:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18382848<>18386944 00000080:00200000:1.0:1789760726.753350:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4488, 4488] 00000080:00200000:1.0:1789760726.753368:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.753374:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.753427:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18386944 i_blocks 36871 00000020:00200000:1.0:1789760726.753430:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.753440:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.753474:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18386944, count: 4096 00000080:00200000:1.0:1789760726.753482:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18386944, bytes: 4096 00000080:00200000:1.0:1789760726.753492:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.753501:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.753503:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18386944, 18391040) 0 0 00000080:00200000:1.0:1789760726.753508:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.753525:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4489, 4489] 00020000:00200000:1.0:1789760726.753533:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18386944 chunk: [18386944, 18391040) 18391040, 4096 00020000:00200000:1.0:1789760726.753538:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.753570:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4489, 4489] 00000008:00010000:1.0:1789760726.753590:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.753595:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.753607:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18386944, 18391040) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.753620:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18386944=0x1189000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.753677:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.753680:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18391040 00000020:00010000:1.0:1789760726.753688:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.753692:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.753695:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.753698:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.753702:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.753710:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.753717:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.753735:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760726.753839:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18386944<>18391040 00000080:00200000:1.0:1789760726.790123:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4489, 4489] 00000080:00200000:1.0:1789760726.790147:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.790153:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.790169:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18391040 i_blocks 36871 00000020:00200000:1.0:1789760726.790173:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.790183:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.790218:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18391040, count: 4096 00000080:00200000:1.0:1789760726.790226:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18391040, bytes: 4096 00000080:00200000:1.0:1789760726.790237:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.790247:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.790249:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18391040, 18395136) 0 0 00000080:00200000:1.0:1789760726.790253:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.790271:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4490, 4490] 00020000:00200000:1.0:1789760726.790279:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18391040 chunk: [18391040, 18395136) 18395136, 4096 00020000:00200000:1.0:1789760726.790284:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.790305:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4490, 4490] 00000008:00010000:1.0:1789760726.790323:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.790329:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.790341:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18391040, 18395136) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.790356:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18391040=0x118a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.790431:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.790434:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18395136 00000020:00010000:1.0:1789760726.790442:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.790445:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.790448:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.790452:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.790456:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.790464:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.790471:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.834214:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4490, 4490] 00000080:00200000:1.0:1789760726.834230:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.834237:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.834255:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18395136 i_blocks 36871 00000020:00200000:1.0:1789760726.834259:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.834269:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.834304:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18395136, count: 4096 00000080:00200000:1.0:1789760726.834311:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18395136, bytes: 4096 00000080:00200000:1.0:1789760726.834321:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.834331:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.834333:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18395136, 18399232) 0 0 00000080:00200000:1.0:1789760726.834338:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.834355:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4491, 4491] 00020000:00200000:1.0:1789760726.834364:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18395136 chunk: [18395136, 18399232) 18399232, 4096 00020000:00200000:1.0:1789760726.834368:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.834425:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4491, 4491] 00000008:00010000:1.0:1789760726.834444:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.834450:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.834461:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18395136, 18399232) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.834476:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18395136=0x118b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.834536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.834552:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18399232 00000020:00010000:1.0:1789760726.834561:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.834564:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.834567:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.834570:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.834573:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.834581:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.834589:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.834608:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760726.834704:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18395136<>18399232 00000080:00200000:1.0:1789760726.873045:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4491, 4491] 00000080:00200000:1.0:1789760726.873061:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760726.873067:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760726.873082:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18399232 i_blocks 36871 00000020:00200000:1.0:1789760726.873087:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760726.873107:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760726.873151:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18399232, count: 4096 00000080:00200000:1.0:1789760726.873159:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18399232, bytes: 4096 00000080:00200000:1.0:1789760726.873168:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760726.873179:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760726.873181:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18399232, 18403328) 0 0 00000080:00200000:1.0:1789760726.873186:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760726.873204:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4492, 4492] 00020000:00200000:1.0:1789760726.873214:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18399232 chunk: [18399232, 18403328) 18403328, 4096 00020000:00200000:1.0:1789760726.873218:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760726.873238:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4492, 4492] 00000008:00010000:1.0:1789760726.873257:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760726.873263:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760726.873275:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18399232, 18403328) append 0 append lockpos 0 00000080:00200000:1.0:1789760726.873288:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18399232=0x118c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760726.873346:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760726.873349:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18403328 00000020:00010000:1.0:1789760726.873357:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760726.873361:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.873364:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760726.873368:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760726.873371:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760726.873424:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760726.873432:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760726.873450:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760726.873565:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18399232<>18403328 00000080:00200000:1.0:1789760727.030164:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4493, 4493] 00000080:00200000:1.0:1789760727.030181:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.030188:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.030204:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18407424 i_blocks 36871 00000020:00200000:1.0:1789760727.030208:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.030219:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.030255:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18407424, count: 4096 00000080:00200000:1.0:1789760727.030263:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18407424, bytes: 4096 00000080:00200000:1.0:1789760727.030274:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.030283:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.030285:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18407424, 18411520) 0 0 00000080:00200000:1.0:1789760727.030290:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.030306:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4494, 4494] 00020000:00200000:1.0:1789760727.030315:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18407424 chunk: [18407424, 18411520) 18411520, 4096 00020000:00200000:1.0:1789760727.030320:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.030341:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4494, 4494] 00000008:00010000:1.0:1789760727.030534:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.030548:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.030561:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18407424, 18411520) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.030579:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18407424=0x118e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.030638:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.030641:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18411520 00000020:00010000:1.0:1789760727.030648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.030652:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.030655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.030658:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.030661:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.030669:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.030676:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.030695:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760727.030806:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18407424<>18411520 00000080:00200000:1.0:1789760727.088344:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4494, 4494] 00000080:00200000:1.0:1789760727.088361:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.088367:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.088416:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18411520 i_blocks 36871 00000020:00200000:1.0:1789760727.088421:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.088432:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.088466:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18411520, count: 4096 00000080:00200000:1.0:1789760727.088473:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18411520, bytes: 4096 00000080:00200000:1.0:1789760727.088482:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.088491:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.088494:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18411520, 18415616) 0 0 00000080:00200000:1.0:1789760727.088498:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.088515:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4495, 4495] 00020000:00200000:1.0:1789760727.088524:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18411520 chunk: [18411520, 18415616) 18415616, 4096 00020000:00200000:1.0:1789760727.088529:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.088561:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4495, 4495] 00000008:00010000:1.0:1789760727.088578:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.088585:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.088596:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18411520, 18415616) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.088609:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18411520=0x118f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.088662:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.088665:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18415616 00000020:00010000:1.0:1789760727.088673:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.088677:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.088680:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.088684:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.088687:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.088694:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.088701:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.088718:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760727.088819:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18411520<>18415616 00000080:00200000:1.0:1789760727.129912:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4495, 4495] 00000080:00200000:1.0:1789760727.129928:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.129935:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.129953:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18415616 i_blocks 36871 00000020:00200000:1.0:1789760727.129957:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.129968:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.130004:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18415616, count: 4096 00000080:00200000:1.0:1789760727.130011:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18415616, bytes: 4096 00000080:00200000:1.0:1789760727.130021:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.130031:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.130033:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18415616, 18419712) 0 0 00000080:00200000:1.0:1789760727.130038:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.130055:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4496, 4496] 00020000:00200000:1.0:1789760727.130064:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18415616 chunk: [18415616, 18419712) 18419712, 4096 00020000:00200000:1.0:1789760727.130069:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.130090:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4496, 4496] 00000008:00010000:1.0:1789760727.130118:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.130124:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.130146:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18415616, 18419712) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.130161:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18415616=0x1190000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.130227:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.130230:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18419712 00000020:00010000:1.0:1789760727.130238:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.130242:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.130245:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.130249:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.130252:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.130260:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.130268:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.130304:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760727.130500:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18415616<>18419712 00000080:00200000:1.0:1789760727.165999:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4496, 4496] 00000080:00200000:1.0:1789760727.166015:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.166021:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.166038:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18419712 i_blocks 36871 00000020:00200000:1.0:1789760727.166042:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.166052:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.166086:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18419712, count: 4096 00000080:00200000:1.0:1789760727.166103:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18419712, bytes: 4096 00000080:00200000:1.0:1789760727.166113:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.166122:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.166124:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18419712, 18423808) 0 0 00000080:00200000:1.0:1789760727.166137:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.166155:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4497, 4497] 00020000:00200000:1.0:1789760727.166164:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18419712 chunk: [18419712, 18423808) 18423808, 4096 00020000:00200000:1.0:1789760727.166169:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.166189:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4497, 4497] 00000008:00010000:1.0:1789760727.166208:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.166214:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.166225:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18419712, 18423808) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.166239:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18419712=0x1191000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.166457:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.166460:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18423808 00000020:00010000:1.0:1789760727.166469:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.166473:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.166476:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.166481:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.166484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.166492:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.166499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.166515:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760727.166686:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 18419712<>18423808 00000080:00200000:1.0:1789760727.199264:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4497, 4497] 00000080:00200000:1.0:1789760727.199280:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.199287:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.199301:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18423808 i_blocks 36871 00000020:00200000:1.0:1789760727.199305:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.199315:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.199350:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18423808, count: 4096 00000080:00200000:1.0:1789760727.199358:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18423808, bytes: 4096 00000080:00200000:1.0:1789760727.199367:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.199410:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.199412:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18423808, 18427904) 0 0 00000080:00200000:1.0:1789760727.199418:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.199435:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4498, 4498] 00020000:00200000:1.0:1789760727.199445:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18423808 chunk: [18423808, 18427904) 18427904, 4096 00020000:00200000:1.0:1789760727.199450:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.199473:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4498, 4498] 00000008:00010000:1.0:1789760727.199496:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.199503:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.199516:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18423808, 18427904) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.199532:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18423808=0x1192000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.199602:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.199605:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18427904 00000020:00010000:1.0:1789760727.199614:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.199618:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.199621:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.199625:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.199628:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.199636:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.199644:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.199661:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760727.199770:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 18423808<>18427904 00000080:00200000:1.0:1789760727.227830:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4498, 4498] 00000080:00200000:1.0:1789760727.227848:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.227854:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.227871:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18427904 i_blocks 36871 00000020:00200000:1.0:1789760727.227875:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.227885:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.227920:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18427904, count: 4096 00000080:00200000:1.0:1789760727.227928:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18427904, bytes: 4096 00000080:00200000:1.0:1789760727.227937:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.227947:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.227949:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18427904, 18432000) 0 0 00000080:00200000:1.0:1789760727.227953:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.227970:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4499, 4499] 00020000:00200000:1.0:1789760727.227979:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18427904 chunk: [18427904, 18432000) 18432000, 4096 00020000:00200000:1.0:1789760727.227983:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.228003:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4499, 4499] 00000008:00010000:1.0:1789760727.228020:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.228025:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.228037:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18427904, 18432000) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.228050:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18427904=0x1193000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.228119:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.228122:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18432000 00000020:00010000:1.0:1789760727.228140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.228144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.228147:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.228150:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.228153:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.228162:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.228169:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.228186:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760727.228288:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 18427904<>18432000 00000080:00200000:1.0:1789760727.261224:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4499, 4499] 00000080:00200000:1.0:1789760727.261241:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.261247:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.261263:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18432000 i_blocks 36871 00000020:00200000:1.0:1789760727.261267:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.261278:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.261313:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18432000, count: 4096 00000080:00200000:1.0:1789760727.261320:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18432000, bytes: 4096 00000080:00200000:1.0:1789760727.261330:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.261339:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.261341:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18432000, 18436096) 0 0 00000080:00200000:1.0:1789760727.261346:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.261364:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4500, 4500] 00020000:00200000:1.0:1789760727.261373:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18432000 chunk: [18432000, 18436096) 18436096, 4096 00020000:00200000:1.0:1789760727.261412:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.261434:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4500, 4500] 00000008:00010000:1.0:1789760727.261452:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.261458:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.261469:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18432000, 18436096) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.261484:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18432000=0x1194000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.261552:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.261555:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18436096 00000020:00010000:1.0:1789760727.261564:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.261567:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.261570:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.261574:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.261576:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.261585:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.261593:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.261610:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760727.261724:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 18432000<>18436096 00000080:00200000:1.0:1789760727.298854:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4500, 4500] 00000080:00200000:1.0:1789760727.298870:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.298877:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.298893:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18436096 i_blocks 36871 00000020:00200000:1.0:1789760727.298909:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.298919:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.298955:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18436096, count: 4096 00000080:00200000:1.0:1789760727.298962:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18436096, bytes: 4096 00000080:00200000:1.0:1789760727.298972:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.298994:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.298997:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18436096, 18440192) 0 0 00000080:00200000:1.0:1789760727.299002:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.299020:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4501, 4501] 00020000:00200000:1.0:1789760727.299028:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18436096 chunk: [18436096, 18440192) 18440192, 4096 00020000:00200000:1.0:1789760727.299033:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.299052:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4501, 4501] 00000008:00010000:1.0:1789760727.299071:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.299077:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.299088:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18436096, 18440192) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.299111:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18436096=0x1195000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.299182:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.299185:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18440192 00000020:00010000:1.0:1789760727.299194:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.299197:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.299201:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.299204:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.299207:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.299214:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.299222:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.299241:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760727.299577:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18436096<>18440192 00000080:00200000:1.0:1789760727.351656:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4501, 4501] 00000080:00200000:1.0:1789760727.351676:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.351682:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.351699:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18440192 i_blocks 36871 00000020:00200000:1.0:1789760727.351702:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.351712:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.351748:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18440192, count: 4096 00000080:00200000:1.0:1789760727.351756:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18440192, bytes: 4096 00000080:00200000:1.0:1789760727.351766:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.351775:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.351778:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18440192, 18444288) 0 0 00000080:00200000:1.0:1789760727.351782:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.354522:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4502, 4502] 00020000:00200000:1.0:1789760727.354602:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18440192 chunk: [18440192, 18444288) 18444288, 4096 00020000:00200000:1.0:1789760727.354608:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.354634:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4502, 4502] 00000008:00010000:1.0:1789760727.354658:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.354664:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.354677:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18440192, 18444288) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.354695:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18440192=0x1196000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.354776:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.354781:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18444288 00000020:00010000:1.0:1789760727.354790:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.354793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.354796:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.354799:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.354802:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.354810:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.354818:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.354843:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760727.355049:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18440192<>18444288 00000080:00200000:1.0:1789760727.385590:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4502, 4502] 00000080:00200000:1.0:1789760727.385607:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.385613:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.385628:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18444288 i_blocks 36871 00000020:00200000:1.0:1789760727.385632:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.385642:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.385682:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18444288, count: 4096 00000080:00200000:1.0:1789760727.385690:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18444288, bytes: 4096 00000080:00200000:1.0:1789760727.385699:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.385709:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.385711:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18444288, 18448384) 0 0 00000080:00200000:1.0:1789760727.385721:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.385740:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4503, 4503] 00020000:00200000:1.0:1789760727.385749:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18444288 chunk: [18444288, 18448384) 18448384, 4096 00020000:00200000:1.0:1789760727.385753:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.385773:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4503, 4503] 00000008:00010000:1.0:1789760727.385793:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.385799:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.385811:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18444288, 18448384) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.385825:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18444288=0x1197000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.385883:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.385886:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18448384 00000020:00010000:1.0:1789760727.385954:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.385959:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.385962:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.385966:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.385968:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.385977:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.385985:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.386003:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760727.386117:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18444288<>18448384 00000080:00200000:1.0:1789760727.425545:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4503, 4503] 00000080:00200000:1.0:1789760727.425563:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.425569:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.425585:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18448384 i_blocks 36871 00000020:00200000:1.0:1789760727.425588:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.425599:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.425634:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18448384, count: 4096 00000080:00200000:1.0:1789760727.425641:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18448384, bytes: 4096 00000080:00200000:1.0:1789760727.425653:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.425662:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.425664:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18448384, 18452480) 0 0 00000080:00200000:1.0:1789760727.425669:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.425685:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4504, 4504] 00020000:00200000:1.0:1789760727.425695:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18448384 chunk: [18448384, 18452480) 18452480, 4096 00020000:00200000:1.0:1789760727.425700:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.425719:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4504, 4504] 00000008:00010000:1.0:1789760727.425739:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.425745:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.425757:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18448384, 18452480) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.425771:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18448384=0x1198000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.425833:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.425836:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18452480 00000020:00010000:1.0:1789760727.425845:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.425848:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.425851:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.425855:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.425858:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.425865:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.425873:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.425894:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760727.426008:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18448384<>18452480 00000080:00200000:1.0:1789760727.455295:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4504, 4504] 00000080:00200000:1.0:1789760727.455316:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.455322:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.455340:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18452480 i_blocks 36871 00000020:00200000:1.0:1789760727.455343:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.455356:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.455430:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18452480, count: 4096 00000080:00200000:1.0:1789760727.455437:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18452480, bytes: 4096 00000080:00200000:1.0:1789760727.455447:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.455456:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.455458:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18452480, 18456576) 0 0 00000080:00200000:1.0:1789760727.455463:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.455480:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4505, 4505] 00020000:00200000:1.0:1789760727.455488:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18452480 chunk: [18452480, 18456576) 18456576, 4096 00020000:00200000:1.0:1789760727.455493:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.455518:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4505, 4505] 00000008:00010000:1.0:1789760727.455602:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.455611:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.455623:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18452480, 18456576) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.455640:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18452480=0x1199000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.455706:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.455709:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18456576 00000020:00010000:1.0:1789760727.455717:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.455722:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.455728:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.455731:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.455734:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.455744:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.455752:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.455772:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760727.455893:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18452480<>18456576 00000080:00200000:1.0:1789760727.492968:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4505, 4505] 00000080:00200000:1.0:1789760727.492985:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.492992:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.493007:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18456576 i_blocks 36871 00000020:00200000:1.0:1789760727.493011:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.493022:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.493061:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18456576, count: 4096 00000080:00200000:1.0:1789760727.493069:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18456576, bytes: 4096 00000080:00200000:1.0:1789760727.493080:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.493101:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.493104:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18456576, 18460672) 0 0 00000080:00200000:1.0:1789760727.493108:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.493126:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4506, 4506] 00020000:00200000:1.0:1789760727.493135:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18456576 chunk: [18456576, 18460672) 18460672, 4096 00020000:00200000:1.0:1789760727.493140:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.493160:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4506, 4506] 00000008:00010000:1.0:1789760727.493179:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.493186:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.493197:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18456576, 18460672) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.493212:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18456576=0x119a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.493270:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.493273:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18460672 00000020:00010000:1.0:1789760727.493282:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.493285:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.493288:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.493292:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.493295:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.493303:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.493310:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.493329:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760727.493653:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18456576<>18460672 00000080:00200000:1.0:1789760727.543205:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4506, 4506] 00000080:00200000:1.0:1789760727.543235:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.543243:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.543264:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18460672 i_blocks 36871 00000020:00200000:1.0:1789760727.543270:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.543285:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.543323:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18460672, count: 4096 00000080:00200000:1.0:1789760727.543332:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18460672, bytes: 4096 00000080:00200000:1.0:1789760727.543347:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.543356:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.543359:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18460672, 18464768) 0 0 00000080:00200000:1.0:1789760727.543363:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.543409:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4507, 4507] 00020000:00200000:1.0:1789760727.543419:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18460672 chunk: [18460672, 18464768) 18464768, 4096 00020000:00200000:1.0:1789760727.543424:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.543451:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4507, 4507] 00000008:00010000:1.0:1789760727.543472:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.543478:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.543491:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18460672, 18464768) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.543507:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18460672=0x119b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.544796:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.544805:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18464768 00000020:00010000:1.0:1789760727.544818:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.544822:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.544825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.544829:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.544831:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.544842:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.544851:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.544887:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760727.545006:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18460672<>18464768 00000008:00010000:1.0:1789760727.595112:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:1.0:1789760727.595189:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760727.595203:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760727.595227:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:1.0:1789760727.595259:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507206400:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760727.802925:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4512, 4512] 00000080:00200000:1.0:1789760727.802942:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760727.802948:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760727.802964:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18485248 i_blocks 36871 00000020:00200000:1.0:1789760727.802967:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760727.802977:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760727.803023:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18485248, count: 4096 00000080:00200000:1.0:1789760727.803030:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18485248, bytes: 4096 00000080:00200000:1.0:1789760727.803040:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760727.803049:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760727.803051:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18485248, 18489344) 0 0 00000080:00200000:1.0:1789760727.803055:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760727.803072:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4513, 4513] 00020000:00200000:1.0:1789760727.803080:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18485248 chunk: [18485248, 18489344) 18489344, 4096 00020000:00200000:1.0:1789760727.803084:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760727.803112:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4513, 4513] 00000008:00010000:1.0:1789760727.803232:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760727.803238:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760727.803250:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18485248, 18489344) append 0 append lockpos 0 00000080:00200000:1.0:1789760727.803264:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18485248=0x11a1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760727.803324:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760727.803327:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18489344 00000020:00010000:1.0:1789760727.803336:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760727.803348:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.803352:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760727.803360:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760727.803364:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760727.803490:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760727.803499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760727.803520:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760727.803630:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18485248<>18489344 00000100:00100000:1.0:1789760727.803670:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760727.803687:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507207168:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760727.872759:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:1.0:1789760727.872775:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760727.872787:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760727.872872:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:1.0:1789760727.872907:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507207168:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760728.032333:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4515, 4515] 00000080:00200000:1.0:1789760728.032359:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.032367:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.032441:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18497536 i_blocks 36871 00000020:00200000:1.0:1789760728.032447:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.032463:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.032503:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18497536, count: 4096 00000080:00200000:1.0:1789760728.032511:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18497536, bytes: 4096 00000080:00200000:1.0:1789760728.032527:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.032536:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.032539:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18497536, 18501632) 0 0 00000080:00200000:1.0:1789760728.032544:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.032563:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4516, 4516] 00020000:00200000:1.0:1789760728.032572:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18497536 chunk: [18497536, 18501632) 18501632, 4096 00020000:00200000:1.0:1789760728.032577:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.032599:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4516, 4516] 00000008:00010000:1.0:1789760728.032622:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.032628:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.032641:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18497536, 18501632) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.032658:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18497536=0x11a4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.032716:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.032719:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18501632 00000020:00010000:1.0:1789760728.032727:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.032730:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.032733:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.032736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.032739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.032746:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.032753:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.032785:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760728.032899:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18497536<>18501632 00000100:00100000:1.0:1789760728.032967:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507207552:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760728.072790:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:1.0:1789760728.072808:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760728.072819:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760728.072905:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:1.0:1789760728.072938:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507207552:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760728.162366:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4518, 4518] 00000080:00200000:1.0:1789760728.162423:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.162430:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.162446:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18509824 i_blocks 36871 00000020:00200000:1.0:1789760728.162449:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.162461:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.162495:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18509824, count: 4096 00000080:00200000:1.0:1789760728.162503:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18509824, bytes: 4096 00000080:00200000:1.0:1789760728.162516:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.162524:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.162527:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18509824, 18513920) 0 0 00000080:00200000:1.0:1789760728.162543:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.162563:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4519, 4519] 00020000:00200000:1.0:1789760728.162573:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18509824 chunk: [18509824, 18513920) 18513920, 4096 00020000:00200000:1.0:1789760728.162578:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.162599:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4519, 4519] 00000008:00010000:1.0:1789760728.162619:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.162626:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.162638:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18509824, 18513920) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.162654:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18509824=0x11a7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.162716:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.162719:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18513920 00000020:00010000:1.0:1789760728.162728:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.162732:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.162736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.162739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.162742:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.162750:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.162757:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.162781:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760728.162892:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18509824<>18513920 00000100:00100000:1.0:1789760728.163124:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760728.163155:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507207936:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760728.194474:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971839 gen 3 00000008:00010000:1.0:1789760728.194538:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:1.0:1789760728.194547:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760728.194560:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760728.194590:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:1.0:1789760728.194621:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507207936:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760728.251175:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4521, 4521] 00000080:00200000:1.0:1789760728.251191:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.251199:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.251214:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18522112 i_blocks 36871 00000020:00200000:1.0:1789760728.251217:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.251228:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.251264:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18522112, count: 4096 00000080:00200000:1.0:1789760728.251271:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18522112, bytes: 4096 00000080:00200000:1.0:1789760728.251280:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.251289:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.251291:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18522112, 18526208) 0 0 00000080:00200000:1.0:1789760728.251296:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.251313:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4522, 4522] 00020000:00200000:1.0:1789760728.251322:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18522112 chunk: [18522112, 18526208) 18526208, 4096 00020000:00200000:1.0:1789760728.251327:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.251348:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4522, 4522] 00000008:00010000:1.0:1789760728.251368:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.251374:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e61330, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.251418:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18522112, 18526208) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.251433:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18522112=0x11aa000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.251493:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.251497:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18526208 00000020:00010000:1.0:1789760728.251506:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.251509:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.251512:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e61330 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.251516:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.251518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e61330 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.251527:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.251598:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.251616:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760728.251719:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18522112<>18526208 00000100:00100000:1.0:1789760728.284725:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971842 gen 3 00000008:00010000:1.0:1789760728.284768:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:1.0:1789760728.284775:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760728.284784:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760728.284802:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:1.0:1789760728.284826:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507208320:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760728.325728:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4523, 4523] 00000080:00200000:1.0:1789760728.325746:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.325752:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.325769:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18530304 i_blocks 36871 00000020:00200000:1.0:1789760728.325773:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.325783:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.325818:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18530304, count: 4096 00000080:00200000:1.0:1789760728.325827:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18530304, bytes: 4096 00000080:00200000:1.0:1789760728.325840:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.325849:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.325851:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18530304, 18534400) 0 0 00000080:00200000:1.0:1789760728.325855:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.325873:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4524, 4524] 00020000:00200000:1.0:1789760728.325882:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18530304 chunk: [18530304, 18534400) 18534400, 4096 00020000:00200000:1.0:1789760728.325887:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.325908:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4524, 4524] 00000100:00100000:1.0:1789760728.359649:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971844 gen 3 00000008:00010000:1.0:1789760728.359698:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760728.359705:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760728.359717:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760728.359739:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760728.359770:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507208576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760728.599703:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507209344:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760728.630346:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760728.630363:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760728.630374:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760728.630643:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760728.630676:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507209344:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760728.631433:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507209472:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760728.669078:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971851 gen 3 00000008:00010000:1.0:1789760728.669152:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760728.669161:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760728.669173:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760728.669232:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760728.669264:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507209472:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760728.675419:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760728.675448:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507209600:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760728.732444:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4532, 4532] 00000080:00200000:1.0:1789760728.732477:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.732484:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.732506:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18567168 i_blocks 36871 00000020:00200000:1.0:1789760728.732512:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.732595:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.732631:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18567168, count: 4096 00000080:00200000:1.0:1789760728.732641:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18567168, bytes: 4096 00000080:00200000:1.0:1789760728.732656:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.732665:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.732668:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18567168, 18571264) 0 0 00000080:00200000:1.0:1789760728.732673:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.732692:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4533, 4533] 00020000:00200000:1.0:1789760728.732701:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18567168 chunk: [18567168, 18571264) 18571264, 4096 00020000:00200000:1.0:1789760728.732706:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.732731:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4533, 4533] 00000008:00010000:1.0:1789760728.732753:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.732759:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.732772:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18567168, 18571264) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.732794:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18567168=0x11b5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.732871:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.732874:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18571264 00000020:00010000:1.0:1789760728.732882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.732886:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.732889:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.732892:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.732895:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.732904:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.732911:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.732945:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760728.733065:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18567168<>18571264 00000080:00200000:1.0:1789760728.765036:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4533, 4533] 00000080:00200000:1.0:1789760728.765052:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.765058:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.765074:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18571264 i_blocks 36871 00000020:00200000:1.0:1789760728.765077:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.765097:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.765137:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18571264, count: 4096 00000080:00200000:1.0:1789760728.765144:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18571264, bytes: 4096 00000080:00200000:1.0:1789760728.765154:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.765162:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.765165:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18571264, 18575360) 0 0 00000080:00200000:1.0:1789760728.765170:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.765185:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4534, 4534] 00020000:00200000:1.0:1789760728.765193:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18571264 chunk: [18571264, 18575360) 18575360, 4096 00020000:00200000:1.0:1789760728.765198:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.765217:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4534, 4534] 00000008:00010000:1.0:1789760728.765236:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.765242:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.765253:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18571264, 18575360) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.765268:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18571264=0x11b6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.765324:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.765326:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18575360 00000020:00010000:1.0:1789760728.765334:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.765338:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.765341:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.765344:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.765347:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.765354:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.765360:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.765374:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760728.765502:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18571264<>18575360 00000080:00200000:1.0:1789760728.802137:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4534, 4534] 00000080:00200000:1.0:1789760728.802155:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.802160:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.802176:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18575360 i_blocks 36871 00000020:00200000:1.0:1789760728.802179:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.802189:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.802225:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18575360, count: 4096 00000080:00200000:1.0:1789760728.802232:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18575360, bytes: 4096 00000080:00200000:1.0:1789760728.802243:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.802252:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.802255:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18575360, 18579456) 0 0 00000080:00200000:1.0:1789760728.802260:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.802275:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4535, 4535] 00020000:00200000:1.0:1789760728.802283:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18575360 chunk: [18575360, 18579456) 18579456, 4096 00020000:00200000:1.0:1789760728.802287:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.802307:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4535, 4535] 00000008:00010000:1.0:1789760728.802326:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.802332:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.802343:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18575360, 18579456) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.802356:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18575360=0x11b7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.802448:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.802452:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18579456 00000020:00010000:1.0:1789760728.802460:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.802464:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.802467:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.802471:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.802473:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.802480:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.802487:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.802503:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760728.802682:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18575360<>18579456 00000080:00200000:1.0:1789760728.841834:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4535, 4535] 00000080:00200000:1.0:1789760728.841853:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.841860:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.841876:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18579456 i_blocks 36871 00000020:00200000:1.0:1789760728.841880:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.841890:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.841937:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18579456, count: 4096 00000080:00200000:1.0:1789760728.841946:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18579456, bytes: 4096 00000080:00200000:1.0:1789760728.841958:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.841967:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.841970:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18579456, 18583552) 0 0 00000080:00200000:1.0:1789760728.841975:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.841992:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4536, 4536] 00020000:00200000:1.0:1789760728.842001:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18579456 chunk: [18579456, 18583552) 18583552, 4096 00020000:00200000:1.0:1789760728.842006:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.842024:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4536, 4536] 00000008:00010000:1.0:1789760728.842041:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.842048:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.842060:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18579456, 18583552) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.842074:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18579456=0x11b8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.842130:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.842133:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18583552 00000020:00010000:1.0:1789760728.842141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.842145:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.842149:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.842152:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.842156:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.842162:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.842168:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.882719:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4536, 4536] 00000080:00200000:1.0:1789760728.882869:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.882876:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.882892:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18583552 i_blocks 36871 00000020:00200000:1.0:1789760728.883003:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.883015:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.883049:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18583552, count: 4096 00000080:00200000:1.0:1789760728.883056:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18583552, bytes: 4096 00000080:00200000:1.0:1789760728.883066:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.883075:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.883077:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18583552, 18587648) 0 0 00000080:00200000:1.0:1789760728.883092:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.883108:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4537, 4537] 00020000:00200000:1.0:1789760728.883117:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18583552 chunk: [18583552, 18587648) 18587648, 4096 00020000:00200000:1.0:1789760728.883122:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.883142:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4537, 4537] 00000008:00010000:1.0:1789760728.883161:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.883167:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.883180:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18583552, 18587648) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.883195:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18583552=0x11b9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.883255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.883258:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18587648 00000020:00010000:1.0:1789760728.883268:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.883272:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.883275:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.883279:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.883282:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.883291:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.883299:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.883319:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760728.883589:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 18583552<>18587648 00000080:00200000:1.0:1789760728.942484:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4537, 4537] 00000080:00200000:1.0:1789760728.942513:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.942519:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.942539:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18587648 i_blocks 36871 00000020:00200000:1.0:1789760728.942545:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.942561:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.942596:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18587648, count: 4096 00000080:00200000:1.0:1789760728.942604:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18587648, bytes: 4096 00000080:00200000:1.0:1789760728.942620:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.942629:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.942631:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18587648, 18591744) 0 0 00000080:00200000:1.0:1789760728.942636:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.942656:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4538, 4538] 00020000:00200000:1.0:1789760728.942666:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18587648 chunk: [18587648, 18591744) 18591744, 4096 00020000:00200000:1.0:1789760728.942671:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.942694:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4538, 4538] 00000008:00010000:1.0:1789760728.942713:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.942720:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.942731:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18587648, 18591744) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.943451:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18587648=0x11ba000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.943533:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.943536:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18591744 00000020:00010000:1.0:1789760728.943546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.943549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.943553:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.943557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.943560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.943570:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.943578:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.943613:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760728.943724:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 18587648<>18591744 00000080:00200000:1.0:1789760728.977027:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4538, 4538] 00000080:00200000:1.0:1789760728.977042:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760728.977048:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760728.977063:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18591744 i_blocks 36871 00000020:00200000:1.0:1789760728.977066:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760728.977075:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760728.977117:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18591744, count: 4096 00000080:00200000:1.0:1789760728.977124:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18591744, bytes: 4096 00000080:00200000:1.0:1789760728.977134:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760728.977142:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760728.977144:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18591744, 18595840) 0 0 00000080:00200000:1.0:1789760728.977149:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760728.977164:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4539, 4539] 00020000:00200000:1.0:1789760728.977172:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18591744 chunk: [18591744, 18595840) 18595840, 4096 00020000:00200000:1.0:1789760728.977176:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760728.977193:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4539, 4539] 00000008:00010000:1.0:1789760728.977210:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760728.977216:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760728.977227:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18591744, 18595840) append 0 append lockpos 0 00000080:00200000:1.0:1789760728.977244:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18591744=0x11bb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760728.977302:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760728.977305:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18595840 00000020:00010000:1.0:1789760728.977314:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760728.977317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.977321:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760728.977324:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760728.977327:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760728.977334:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760728.977341:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760728.977357:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760728.977653:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 18591744<>18595840 00000080:00200000:1.0:1789760729.010264:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4539, 4539] 00000080:00200000:1.0:1789760729.010281:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760729.010286:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760729.010304:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18595840 i_blocks 36871 00000020:00200000:1.0:1789760729.010307:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760729.010316:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760729.010351:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18595840, count: 4096 00000080:00200000:1.0:1789760729.010358:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18595840, bytes: 4096 00000080:00200000:1.0:1789760729.010368:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760729.010406:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760729.010409:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18595840, 18599936) 0 0 00000080:00200000:1.0:1789760729.010414:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760729.010430:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4540, 4540] 00020000:00200000:1.0:1789760729.010439:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18595840 chunk: [18595840, 18599936) 18599936, 4096 00020000:00200000:1.0:1789760729.010443:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760729.010463:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4540, 4540] 00000008:00010000:1.0:1789760729.010482:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760729.010487:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760729.010499:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18595840, 18599936) append 0 append lockpos 0 00000080:00200000:1.0:1789760729.010626:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18595840=0x11bc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760729.010691:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760729.010694:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18599936 00000020:00010000:1.0:1789760729.010703:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760729.010707:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.010710:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.010714:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760729.010717:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760729.010725:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760729.010732:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760729.010749:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760729.010852:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 18595840<>18599936 00000080:00200000:1.0:1789760729.044810:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4540, 4540] 00000080:00200000:1.0:1789760729.044825:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760729.044830:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760729.044845:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18599936 i_blocks 36871 00000020:00200000:1.0:1789760729.044849:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760729.044857:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760729.044889:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18599936, count: 4096 00000080:00200000:1.0:1789760729.044904:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18599936, bytes: 4096 00000080:00200000:1.0:1789760729.044913:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760729.044922:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760729.044924:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18599936, 18604032) 0 0 00000080:00200000:1.0:1789760729.044928:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760729.044944:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4541, 4541] 00020000:00200000:1.0:1789760729.044952:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18599936 chunk: [18599936, 18604032) 18604032, 4096 00020000:00200000:1.0:1789760729.044957:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760729.044975:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4541, 4541] 00000008:00010000:1.0:1789760729.044991:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760729.044996:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760729.045006:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18599936, 18604032) append 0 append lockpos 0 00000080:00200000:1.0:1789760729.045019:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18599936=0x11bd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760729.045071:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760729.045074:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18604032 00000020:00010000:1.0:1789760729.045092:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760729.045095:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.045098:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.045102:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760729.045104:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760729.045112:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760729.045124:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760729.045140:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760729.045234:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18599936<>18604032 00000080:00200000:1.0:1789760729.081163:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4541, 4541] 00000080:00200000:1.0:1789760729.081179:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760729.081186:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760729.081201:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18604032 i_blocks 36871 00000020:00200000:1.0:1789760729.081205:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760729.081214:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760729.081249:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18604032, count: 4096 00000080:00200000:1.0:1789760729.081256:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18604032, bytes: 4096 00000080:00200000:1.0:1789760729.081265:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760729.081274:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760729.081276:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18604032, 18608128) 0 0 00000080:00200000:1.0:1789760729.081280:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760729.081299:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4542, 4542] 00020000:00200000:1.0:1789760729.081308:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18604032 chunk: [18604032, 18608128) 18608128, 4096 00020000:00200000:1.0:1789760729.081313:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760729.081333:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4542, 4542] 00000008:00010000:1.0:1789760729.081351:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760729.081356:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760729.081368:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18604032, 18608128) append 0 append lockpos 0 00000080:00200000:1.0:1789760729.081415:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18604032=0x11be000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760729.081474:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760729.081477:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18608128 00000020:00010000:1.0:1789760729.081486:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760729.081490:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.081493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.081496:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760729.081499:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760729.081507:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760729.081514:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760729.081538:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760729.081643:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18604032<>18608128 00000080:00200000:1.0:1789760729.114117:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4542, 4542] 00000080:00200000:1.0:1789760729.114135:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760729.114141:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760729.114157:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18608128 i_blocks 36871 00000020:00200000:1.0:1789760729.114160:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760729.114170:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760729.114207:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18608128, count: 4096 00000080:00200000:1.0:1789760729.114213:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18608128, bytes: 4096 00000080:00200000:1.0:1789760729.114223:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760729.114233:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760729.114235:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18608128, 18612224) 0 0 00000080:00200000:1.0:1789760729.114240:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760729.114256:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4543, 4543] 00020000:00200000:1.0:1789760729.114264:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18608128 chunk: [18608128, 18612224) 18612224, 4096 00020000:00200000:1.0:1789760729.114268:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760729.114287:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4543, 4543] 00000008:00010000:1.0:1789760729.114305:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760729.114311:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760729.114322:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18608128, 18612224) append 0 append lockpos 0 00000080:00200000:1.0:1789760729.114335:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18608128=0x11bf000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760729.114432:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760729.114435:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18612224 00000020:00010000:1.0:1789760729.114444:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760729.114447:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.114450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.114454:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760729.114457:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760729.114465:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760729.114472:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760729.114489:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451900 00000008:00100000:1.0:1789760729.114622:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18608128<>18612224 00000080:00200000:1.0:1789760729.144969:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4543, 4543] 00000080:00200000:1.0:1789760729.144989:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760729.144995:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760729.145012:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18612224 i_blocks 36871 00000020:00200000:1.0:1789760729.145016:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760729.145026:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760729.145062:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18612224, count: 4096 00000080:00200000:1.0:1789760729.145069:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18612224, bytes: 4096 00000080:00200000:1.0:1789760729.145088:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760729.145097:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760729.145099:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18612224, 18616320) 0 0 00000080:00200000:1.0:1789760729.145103:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760729.145185:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4544, 4544] 00020000:00200000:1.0:1789760729.145194:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18612224 chunk: [18612224, 18616320) 18616320, 4096 00020000:00200000:1.0:1789760729.145199:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760729.145217:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4544, 4544] 00000008:00010000:1.0:1789760729.145234:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760729.145240:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760729.145250:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18612224, 18616320) append 0 append lockpos 0 00000080:00200000:1.0:1789760729.145263:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18612224=0x11c0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760729.145315:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760729.145318:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18616320 00000020:00010000:1.0:1789760729.145326:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760729.145330:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.145332:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.145336:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760729.145339:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760729.145345:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760729.145518:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760729.145536:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760729.145641:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18612224<>18616320 00000100:00100000:1.0:1789760729.145692:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507211136:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760729.332034:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4547, 4547] 00000080:00200000:1.0:1789760729.332054:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760729.332061:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760729.332089:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18628608 i_blocks 36871 00000020:00200000:1.0:1789760729.332095:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760729.332108:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760729.332144:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18628608, count: 4096 00000080:00200000:1.0:1789760729.332153:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18628608, bytes: 4096 00000080:00200000:1.0:1789760729.332167:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760729.332176:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760729.332189:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18628608, 18632704) 0 0 00000080:00200000:1.0:1789760729.332194:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760729.332211:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4548, 4548] 00020000:00200000:1.0:1789760729.332218:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18628608 chunk: [18628608, 18632704) 18632704, 4096 00020000:00200000:1.0:1789760729.332222:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760729.332243:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4548, 4548] 00000008:00010000:1.0:1789760729.332261:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760729.332268:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760729.332280:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18628608, 18632704) append 0 append lockpos 0 00000080:00200000:1.0:1789760729.332294:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18628608=0x11c4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760729.332362:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760729.332364:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18632704 00000020:00010000:1.0:1789760729.332373:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9218 (ffff8d25c8038cf8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760729.332404:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9218 (ffff8d25c8038cf8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.332408:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760729.332412:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9218 (ffff8d25c8038cf8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760729.332415:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760729.332424:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760729.332430:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760729.332456:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760729.332815:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 18628608<>18632704 00000100:00100000:1.0:1789760729.332858:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760729.332882:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507211648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760730.812920:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760730.812937:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507215232:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760730.861995:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4576, 4576] 00000080:00200000:1.0:1789760730.862009:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760730.862014:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760730.862026:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18747392 i_blocks 36871 00000020:00200000:1.0:1789760730.862030:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760730.862039:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760730.862062:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18747392, count: 4096 00000080:00200000:1.0:1789760730.862079:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18747392, bytes: 4096 00000080:00200000:1.0:1789760730.862087:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760730.862095:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760730.862097:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18747392, 18751488) 0 0 00000080:00200000:1.0:1789760730.862100:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760730.862115:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4577, 4577] 00020000:00200000:1.0:1789760730.862122:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18747392 chunk: [18747392, 18751488) 18751488, 4096 00020000:00200000:1.0:1789760730.862127:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760730.862142:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4577, 4577] 00000008:00010000:1.0:1789760730.862165:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760730.862170:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760730.862179:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18747392, 18751488) append 0 append lockpos 0 00000080:00200000:1.0:1789760730.862189:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18747392=0x11e1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760730.862229:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760730.862232:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18751488 00000020:00010000:1.0:1789760730.862240:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760730.862243:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760730.862246:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760730.862249:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760730.862252:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760730.862258:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760730.862263:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760730.862277:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760730.862357:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 18747392<>18751488 00000100:00100000:1.0:1789760730.862429:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760730.862446:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507215360:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760730.909081:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4577, 4577] 00000080:00200000:1.0:1789760730.909093:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760730.909100:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760730.909111:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18751488 i_blocks 36871 00000020:00200000:1.0:1789760730.909115:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760730.909123:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760730.909149:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18751488, count: 4096 00000080:00200000:1.0:1789760730.909155:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18751488, bytes: 4096 00000080:00200000:1.0:1789760730.909163:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760730.909170:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760730.909173:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18751488, 18755584) 0 0 00000080:00200000:1.0:1789760730.909177:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760730.909188:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4578, 4578] 00020000:00200000:1.0:1789760730.909196:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18751488 chunk: [18751488, 18755584) 18755584, 4096 00020000:00200000:1.0:1789760730.909200:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760730.909214:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4578, 4578] 00000008:00010000:1.0:1789760730.909228:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760730.909233:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760730.909242:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18751488, 18755584) append 0 append lockpos 0 00000080:00200000:1.0:1789760730.909253:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18751488=0x11e2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760730.909295:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760730.909298:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18755584 00000020:00010000:1.0:1789760730.909304:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760730.909308:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760730.909311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760730.909314:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760730.909317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760730.909324:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760730.909330:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760730.909343:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760730.909719:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 18751488<>18755584 00000080:00200000:1.0:1789760730.973400:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4578, 4578] 00000080:00200000:1.0:1789760730.973414:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760730.973418:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760730.973430:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18755584 i_blocks 36871 00000020:00200000:1.0:1789760730.973433:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760730.973442:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760730.973470:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18755584, count: 4096 00000080:00200000:1.0:1789760730.973477:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18755584, bytes: 4096 00000080:00200000:1.0:1789760730.973486:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760730.973493:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760730.973495:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18755584, 18759680) 0 0 00000080:00200000:1.0:1789760730.973499:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760730.973522:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4579, 4579] 00020000:00200000:1.0:1789760730.973528:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18755584 chunk: [18755584, 18759680) 18759680, 4096 00020000:00200000:1.0:1789760730.973531:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760730.973547:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4579, 4579] 00000008:00010000:1.0:1789760730.973559:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760730.973563:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760730.973572:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18755584, 18759680) append 0 append lockpos 0 00000080:00200000:1.0:1789760730.973582:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18755584=0x11e3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760730.973625:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760730.973627:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18759680 00000020:00010000:1.0:1789760730.973634:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760730.973637:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760730.973639:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760730.973642:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760730.973645:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760730.973651:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760730.973656:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760730.973671:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760730.973740:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 18755584<>18759680 00000080:00200000:1.0:1789760731.032725:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4579, 4579] 00000080:00200000:1.0:1789760731.032738:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.032743:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.032755:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18759680 i_blocks 36871 00000020:00200000:1.0:1789760731.032758:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.032766:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.032792:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18759680, count: 4096 00000080:00200000:1.0:1789760731.032797:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18759680, bytes: 4096 00000080:00200000:1.0:1789760731.032805:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.032813:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.032815:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18759680, 18763776) 0 0 00000080:00200000:1.0:1789760731.032819:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.032831:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4580, 4580] 00020000:00200000:1.0:1789760731.032837:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18759680 chunk: [18759680, 18763776) 18763776, 4096 00020000:00200000:1.0:1789760731.032841:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.032856:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4580, 4580] 00000008:00010000:1.0:1789760731.032869:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.032873:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.032882:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18759680, 18763776) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.032894:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18759680=0x11e4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.032947:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.032950:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18763776 00000020:00010000:1.0:1789760731.032957:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.032960:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.032963:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.032965:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.032968:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.032974:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.032980:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.032994:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760731.033313:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 18759680<>18763776 00000080:00200000:1.0:1789760731.065138:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4580, 4580] 00000080:00200000:1.0:1789760731.065151:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.065155:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.065165:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18763776 i_blocks 36871 00000020:00200000:1.0:1789760731.065167:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.065174:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.065205:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18763776, count: 4096 00000080:00200000:1.0:1789760731.065209:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18763776, bytes: 4096 00000080:00200000:1.0:1789760731.065215:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.065220:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.065222:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18763776, 18767872) 0 0 00000080:00200000:1.0:1789760731.065225:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.065331:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4581, 4581] 00020000:00200000:1.0:1789760731.065337:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18763776 chunk: [18763776, 18767872) 18767872, 4096 00020000:00200000:1.0:1789760731.065340:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.065353:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4581, 4581] 00000008:00010000:1.0:1789760731.065364:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.065597:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.065608:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18763776, 18767872) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.065621:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18763776=0x11e5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.065665:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.065668:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18767872 00000020:00010000:1.0:1789760731.065675:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.065678:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.065681:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.065684:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.065686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.065692:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.065698:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.065712:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760731.065792:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 18763776<>18767872 00000080:00200000:1.0:1789760731.109127:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4581, 4581] 00000080:00200000:1.0:1789760731.109140:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.109145:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.109157:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18767872 i_blocks 36871 00000020:00200000:1.0:1789760731.109160:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.109169:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.109195:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18767872, count: 4096 00000080:00200000:1.0:1789760731.109202:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18767872, bytes: 4096 00000080:00200000:1.0:1789760731.109209:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.109216:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.109218:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18767872, 18771968) 0 0 00000080:00200000:1.0:1789760731.109222:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.109616:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4582, 4582] 00020000:00200000:1.0:1789760731.109625:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18767872 chunk: [18767872, 18771968) 18771968, 4096 00020000:00200000:1.0:1789760731.109629:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.109645:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4582, 4582] 00000008:00010000:1.0:1789760731.109660:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.109665:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.109673:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18767872, 18771968) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.109684:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18767872=0x11e6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.109731:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.109733:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18771968 00000020:00010000:1.0:1789760731.109739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.109742:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.109744:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.109747:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.109750:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.109756:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.109761:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.109776:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760731.110116:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 18767872<>18771968 00000080:00200000:1.0:1789760731.186812:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4582, 4582] 00000080:00200000:1.0:1789760731.186835:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.186841:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.186935:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18771968 i_blocks 36871 00000020:00200000:1.0:1789760731.186940:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.186953:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.186979:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18771968, count: 4096 00000080:00200000:1.0:1789760731.186986:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18771968, bytes: 4096 00000080:00200000:1.0:1789760731.186997:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.187005:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.187007:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18771968, 18776064) 0 0 00000080:00200000:1.0:1789760731.187011:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.187024:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4583, 4583] 00020000:00200000:1.0:1789760731.187238:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18771968 chunk: [18771968, 18776064) 18776064, 4096 00020000:00200000:1.0:1789760731.187244:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.187261:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4583, 4583] 00000008:00010000:1.0:1789760731.187275:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.187280:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.187289:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18771968, 18776064) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.187301:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18771968=0x11e7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.187355:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.187357:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18776064 00000020:00010000:1.0:1789760731.187365:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.187368:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.187371:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.187374:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.188122:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.188135:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.188142:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.188238:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760731.188320:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 18771968<>18776064 00000080:00200000:1.0:1789760731.228088:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4583, 4583] 00000080:00200000:1.0:1789760731.228101:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.228106:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.228118:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18776064 i_blocks 36871 00000020:00200000:1.0:1789760731.228121:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.228129:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.228156:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18776064, count: 4096 00000080:00200000:1.0:1789760731.228162:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18776064, bytes: 4096 00000080:00200000:1.0:1789760731.228171:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.228178:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.228180:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18776064, 18780160) 0 0 00000080:00200000:1.0:1789760731.228184:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.228196:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4584, 4584] 00020000:00200000:1.0:1789760731.228203:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18776064 chunk: [18776064, 18780160) 18780160, 4096 00020000:00200000:1.0:1789760731.228206:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.228221:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4584, 4584] 00000008:00010000:1.0:1789760731.228235:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.228239:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.228248:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18776064, 18780160) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.228258:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18776064=0x11e8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.228301:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.228304:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18780160 00000020:00010000:1.0:1789760731.228311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.228314:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.228317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.228320:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.228323:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.228330:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.228336:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.228349:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760731.228478:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18776064<>18780160 00000080:00200000:1.0:1789760731.276865:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4584, 4584] 00000080:00200000:1.0:1789760731.276878:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.276883:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.276895:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18780160 i_blocks 36871 00000020:00200000:1.0:1789760731.276898:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.276906:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.276933:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18780160, count: 4096 00000080:00200000:1.0:1789760731.276938:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18780160, bytes: 4096 00000080:00200000:1.0:1789760731.276948:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.276954:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.276957:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18780160, 18784256) 0 0 00000080:00200000:1.0:1789760731.276960:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.276973:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4585, 4585] 00020000:00200000:1.0:1789760731.276979:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18780160 chunk: [18780160, 18784256) 18784256, 4096 00020000:00200000:1.0:1789760731.276983:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.276998:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4585, 4585] 00000008:00010000:1.0:1789760731.277013:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.277017:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.277037:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18780160, 18784256) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.277048:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18780160=0x11e9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.277100:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.277104:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18784256 00000020:00010000:1.0:1789760731.277110:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.277114:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.277117:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.277120:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.277123:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.277130:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.277139:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.277153:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760731.277224:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18780160<>18784256 00000080:00200000:1.0:1789760731.333280:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4585, 4585] 00000080:00200000:1.0:1789760731.333303:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.333309:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.333323:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18784256 i_blocks 36871 00000020:00200000:1.0:1789760731.333327:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.333339:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.333364:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18784256, count: 4096 00000080:00200000:1.0:1789760731.333371:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18784256, bytes: 4096 00000080:00200000:1.0:1789760731.333409:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.333417:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.333419:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18784256, 18788352) 0 0 00000080:00200000:1.0:1789760731.333424:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.333436:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4586, 4586] 00020000:00200000:1.0:1789760731.333443:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18784256 chunk: [18784256, 18788352) 18788352, 4096 00020000:00200000:1.0:1789760731.333447:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.333462:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4586, 4586] 00000008:00010000:1.0:1789760731.333475:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.333480:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.333493:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18784256, 18788352) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.333505:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18784256=0x11ea000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.333556:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.333559:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18788352 00000020:00010000:1.0:1789760731.333566:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.333569:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.333572:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.333575:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.333583:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.333591:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.333597:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.333615:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760731.333691:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18784256<>18788352 00000080:00200000:1.0:1789760731.376056:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4586, 4586] 00000080:00200000:1.0:1789760731.376077:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.376082:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.376093:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18788352 i_blocks 36871 00000020:00200000:1.0:1789760731.376096:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.376113:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.376141:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18788352, count: 4096 00000080:00200000:1.0:1789760731.376147:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18788352, bytes: 4096 00000080:00200000:1.0:1789760731.376154:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.376160:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.376162:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18788352, 18792448) 0 0 00000080:00200000:1.0:1789760731.376166:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.376178:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4587, 4587] 00020000:00200000:1.0:1789760731.376184:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18788352 chunk: [18788352, 18792448) 18792448, 4096 00020000:00200000:1.0:1789760731.376188:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.376203:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4587, 4587] 00000008:00010000:1.0:1789760731.376217:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.376222:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.376232:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18788352, 18792448) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.376243:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18788352=0x11eb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.376287:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.376290:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18792448 00000020:00010000:1.0:1789760731.376297:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.376300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.376304:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.376306:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.376309:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.376315:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.376320:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.376334:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760731.376441:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18788352<>18792448 00000080:00200000:1.0:1789760731.415621:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4587, 4587] 00000080:00200000:1.0:1789760731.415634:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.415639:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.415652:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18792448 i_blocks 36871 00000020:00200000:1.0:1789760731.415656:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.415664:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.415690:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18792448, count: 4096 00000080:00200000:1.0:1789760731.415697:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18792448, bytes: 4096 00000080:00200000:1.0:1789760731.415704:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.415711:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.415712:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18792448, 18796544) 0 0 00000080:00200000:1.0:1789760731.415716:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.415730:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4588, 4588] 00020000:00200000:1.0:1789760731.415736:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18792448 chunk: [18792448, 18796544) 18796544, 4096 00020000:00200000:1.0:1789760731.415740:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.415754:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4588, 4588] 00000008:00010000:1.0:1789760731.415769:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.415772:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.415780:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18792448, 18796544) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.415791:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18792448=0x11ec000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.415833:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.415836:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18796544 00000020:00010000:1.0:1789760731.415843:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.415847:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.415850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.415852:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.415855:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.415861:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.415866:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.415878:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760731.415950:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18792448<>18796544 00000080:00200000:1.0:1789760731.464425:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4588, 4588] 00000080:00200000:1.0:1789760731.464438:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.464444:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.464467:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18796544 i_blocks 36871 00000020:00200000:1.0:1789760731.464471:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.464480:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.464517:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18796544, count: 4096 00000080:00200000:1.0:1789760731.464523:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18796544, bytes: 4096 00000080:00200000:1.0:1789760731.464531:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.464538:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.464540:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18796544, 18800640) 0 0 00000080:00200000:1.0:1789760731.464545:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.464557:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4589, 4589] 00020000:00200000:1.0:1789760731.464564:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18796544 chunk: [18796544, 18800640) 18800640, 4096 00020000:00200000:1.0:1789760731.464568:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.464582:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4589, 4589] 00000008:00010000:1.0:1789760731.464597:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.464601:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.464610:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18796544, 18800640) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.464621:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18796544=0x11ed000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.464666:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.464669:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18800640 00000020:00010000:1.0:1789760731.464676:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.464679:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.464683:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.464686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.464689:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.464696:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.464714:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.464729:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760731.464804:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18796544<>18800640 00000080:00200000:1.0:1789760731.545526:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4589, 4589] 00000080:00200000:1.0:1789760731.545551:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.545557:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.545569:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18800640 i_blocks 36871 00000020:00200000:1.0:1789760731.545572:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.545581:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.545608:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18800640, count: 4096 00000080:00200000:1.0:1789760731.545613:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18800640, bytes: 4096 00000080:00200000:1.0:1789760731.545621:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.545629:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.545631:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18800640, 18804736) 0 0 00000080:00200000:1.0:1789760731.545636:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.545650:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4590, 4590] 00020000:00200000:1.0:1789760731.545657:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18800640 chunk: [18800640, 18804736) 18804736, 4096 00020000:00200000:1.0:1789760731.545661:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.545676:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4590, 4590] 00000008:00010000:1.0:1789760731.545690:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.545695:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.545704:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18800640, 18804736) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.545715:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18800640=0x11ee000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.545764:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.545767:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18804736 00000020:00010000:1.0:1789760731.545773:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.545776:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.545779:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.545782:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.545784:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.545791:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.545796:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.545810:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760731.545889:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 18800640<>18804736 00000080:00200000:1.0:1789760731.611866:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4590, 4590] 00000080:00200000:1.0:1789760731.611879:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.611885:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.611898:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18804736 i_blocks 36871 00000020:00200000:1.0:1789760731.611902:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.611910:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.611937:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18804736, count: 4096 00000080:00200000:1.0:1789760731.611943:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18804736, bytes: 4096 00000080:00200000:1.0:1789760731.611952:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.611959:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.611961:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18804736, 18808832) 0 0 00000080:00200000:1.0:1789760731.611965:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.611977:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4591, 4591] 00020000:00200000:1.0:1789760731.611984:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18804736 chunk: [18804736, 18808832) 18808832, 4096 00020000:00200000:1.0:1789760731.611987:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.612003:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4591, 4591] 00000008:00010000:1.0:1789760731.612029:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.612034:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.612044:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18804736, 18808832) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.612055:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18804736=0x11ef000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.612108:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.612111:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18808832 00000020:00010000:1.0:1789760731.612119:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.612241:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.612245:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.612248:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.612251:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.612259:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.612264:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.612278:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760731.612363:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 18804736<>18808832 00000080:00200000:1.0:1789760731.666610:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4591, 4591] 00000080:00200000:1.0:1789760731.666623:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.666629:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.666640:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18808832 i_blocks 36871 00000020:00200000:1.0:1789760731.666644:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.666652:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.666677:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18808832, count: 4096 00000080:00200000:1.0:1789760731.666683:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18808832, bytes: 4096 00000080:00200000:1.0:1789760731.666692:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.666699:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.666701:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18808832, 18812928) 0 0 00000080:00200000:1.0:1789760731.666705:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.666717:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4592, 4592] 00020000:00200000:1.0:1789760731.666724:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18808832 chunk: [18808832, 18812928) 18812928, 4096 00020000:00200000:1.0:1789760731.666727:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.666745:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4592, 4592] 00000008:00010000:1.0:1789760731.666760:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.666765:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.666775:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18808832, 18812928) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.667193:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18808832=0x11f0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.667256:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.667259:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18812928 00000020:00010000:1.0:1789760731.667267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.667270:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.667273:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.667276:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.667278:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.667285:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.667291:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.667308:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760731.668664:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 18808832<>18812928 00000100:00100000:1.0:1789760731.668725:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760731.668745:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507217280:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760731.783492:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4593, 4593] 00000080:00200000:1.0:1789760731.783512:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.783517:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.783531:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18817024 i_blocks 36871 00000020:00200000:1.0:1789760731.783535:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.783546:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.783582:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18817024, count: 4096 00000080:00200000:1.0:1789760731.783590:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18817024, bytes: 4096 00000080:00200000:1.0:1789760731.783598:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.783606:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.783608:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18817024, 18821120) 0 0 00000080:00200000:1.0:1789760731.783612:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.783626:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4594, 4594] 00020000:00200000:1.0:1789760731.783632:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18817024 chunk: [18817024, 18821120) 18821120, 4096 00020000:00200000:1.0:1789760731.783636:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.783651:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4594, 4594] 00000008:00010000:1.0:1789760731.783665:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.783670:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.783679:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18817024, 18821120) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.783689:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18817024=0x11f2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.783831:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.784000:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18821120 00000020:00010000:1.0:1789760731.784010:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.784013:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.784016:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.784019:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.784021:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.784029:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.784034:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.784067:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760731.784149:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 18817024<>18821120 00000080:00200000:1.0:1789760731.837203:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4594, 4594] 00000080:00200000:1.0:1789760731.837217:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.837223:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.837234:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18821120 i_blocks 36871 00000020:00200000:1.0:1789760731.837238:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.837246:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.837271:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18821120, count: 4096 00000080:00200000:1.0:1789760731.837278:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18821120, bytes: 4096 00000080:00200000:1.0:1789760731.837287:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.837294:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.837296:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18821120, 18825216) 0 0 00000080:00200000:1.0:1789760731.837300:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.837313:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4595, 4595] 00020000:00200000:1.0:1789760731.837319:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18821120 chunk: [18821120, 18825216) 18825216, 4096 00020000:00200000:1.0:1789760731.837322:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.837337:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4595, 4595] 00000008:00010000:1.0:1789760731.837352:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.837356:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.837365:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18821120, 18825216) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.837407:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18821120=0x11f3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.837453:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.837456:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18825216 00000020:00010000:1.0:1789760731.837463:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.837466:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.837469:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.837472:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.837474:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.837481:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.837486:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.837498:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760731.837575:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18821120<>18825216 00000080:00200000:1.0:1789760731.898126:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4595, 4595] 00000080:00200000:1.0:1789760731.898140:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760731.898146:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760731.898158:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18825216 i_blocks 36871 00000020:00200000:1.0:1789760731.898162:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760731.898171:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760731.898199:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18825216, count: 4096 00000080:00200000:1.0:1789760731.898205:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18825216, bytes: 4096 00000080:00200000:1.0:1789760731.898214:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760731.898221:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760731.898223:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18825216, 18829312) 0 0 00000080:00200000:1.0:1789760731.898227:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760731.898239:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4596, 4596] 00020000:00200000:1.0:1789760731.898245:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18825216 chunk: [18825216, 18829312) 18829312, 4096 00020000:00200000:1.0:1789760731.898249:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760731.898263:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4596, 4596] 00000008:00010000:1.0:1789760731.898277:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760731.898282:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760731.898291:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18825216, 18829312) append 0 append lockpos 0 00000080:00200000:1.0:1789760731.898302:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18825216=0x11f4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760731.898348:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760731.898352:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18829312 00000020:00010000:1.0:1789760731.898359:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760731.898362:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.898365:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760731.898368:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760731.898370:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760731.898410:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760731.898417:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760731.898433:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760731.898512:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18825216<>18829312 00000100:00100000:1.0:1789760731.952777:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971915 gen 3 00000100:00100000:1.0:1789760731.952809:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c816ad80 x1876699507217792/t4294971916(4294971916) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760747 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760731.952846:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760731.952852:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760731.952860:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760731.952878:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760731.952897:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507217792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760731.999030:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971917 gen 3 00000008:00010000:1.0:1789760731.999087:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760731.999093:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760731.999103:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760731.999123:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760731.999147:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507218560:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.000741:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760732.000759:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507218688:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.037252:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971918 gen 3 00000008:00010000:1.0:1789760732.037306:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760732.037315:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.037329:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.037349:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760732.037711:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507218688:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.040330:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507218816:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760732.120142:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760732.120159:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.120173:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.120197:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760732.120230:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507218816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.123216:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760732.123263:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507218944:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.170945:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971919 gen 3 00000100:00100000:1.0:1789760732.170988:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b8700 x1876699507218944/t4294971920(4294971920) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760748 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760732.171037:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760732.171044:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.171170:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.171192:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760732.171867:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507218944:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.174972:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507219072:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.217776:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971921 gen 3 00000008:00010000:1.0:1789760732.217848:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760732.217857:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.217869:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.217890:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760732.217920:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507219072:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760732.252146:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971922 gen 3 00000008:00010000:1.0:1789760732.252204:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760732.252213:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.252227:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.252259:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760732.252290:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507219200:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760732.294340:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4603, 4603] 00000080:00200000:1.0:1789760732.294356:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760732.294362:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760732.294650:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18857984 i_blocks 36871 00000020:00200000:1.0:1789760732.294655:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760732.294666:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760732.294695:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18857984, count: 4096 00000080:00200000:1.0:1789760732.294702:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18857984, bytes: 4096 00000080:00200000:1.0:1789760732.294712:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760732.294721:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760732.294723:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18857984, 18862080) 0 0 00000080:00200000:1.0:1789760732.294728:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760732.294745:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4604, 4604] 00020000:00200000:1.0:1789760732.294753:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18857984 chunk: [18857984, 18862080) 18862080, 4096 00020000:00200000:1.0:1789760732.294758:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760732.294776:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4604, 4604] 00000008:00010000:1.0:1789760732.294793:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760732.294799:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760732.294811:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18857984, 18862080) append 0 append lockpos 0 00000080:00200000:1.0:1789760732.294826:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18857984=0x11fc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760732.294890:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760732.294893:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18862080 00000020:00010000:1.0:1789760732.294901:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760732.294905:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760732.294908:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760732.294912:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760732.294914:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760732.294923:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760732.294930:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760732.294949:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760732.295063:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18857984<>18862080 00000100:00100000:1.0:1789760732.323748:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971924 gen 3 00000008:00010000:1.0:1789760732.323789:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760732.323795:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.323803:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.323902:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760732.323937:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507219456:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760732.377756:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4605, 4605] 00000080:00200000:1.0:1789760732.377770:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760732.377776:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760732.377788:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18866176 i_blocks 36871 00000020:00200000:1.0:1789760732.377792:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760732.377799:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760732.377825:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18866176, count: 4096 00000080:00200000:1.0:1789760732.377831:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18866176, bytes: 4096 00000080:00200000:1.0:1789760732.377842:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760732.377849:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760732.377851:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18866176, 18870272) 0 0 00000080:00200000:1.0:1789760732.377854:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760732.377868:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4606, 4606] 00020000:00200000:1.0:1789760732.377875:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18866176 chunk: [18866176, 18870272) 18870272, 4096 00020000:00200000:1.0:1789760732.377879:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760732.377893:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4606, 4606] 00000008:00010000:1.0:1789760732.377905:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760732.377910:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760732.377918:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18866176, 18870272) append 0 append lockpos 0 00000080:00200000:1.0:1789760732.378505:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18866176=0x11fe000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760732.378688:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760732.378692:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18870272 00000020:00010000:1.0:1789760732.378700:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760732.378703:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760732.378707:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760732.378710:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760732.378713:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760732.378720:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760732.378726:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760732.378743:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760732.378820:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18866176<>18870272 00000100:00100000:1.0:1789760732.433687:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971926 gen 3 00000008:00010000:1.0:1789760732.433724:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760732.433730:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.433738:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.433807:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760732.433831:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507219712:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760732.728029:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4610, 4610] 00000080:00200000:1.0:1789760732.728047:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760732.728063:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760732.728081:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18886656 i_blocks 38919 00000020:00200000:1.0:1789760732.728085:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760732.728156:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760732.728193:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18886656, count: 4096 00000080:00200000:1.0:1789760732.728201:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18886656, bytes: 4096 00000080:00200000:1.0:1789760732.728211:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760732.728220:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760732.728223:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18886656, 18890752) 0 0 00000080:00200000:1.0:1789760732.728228:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760732.728246:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4611, 4611] 00020000:00200000:1.0:1789760732.728255:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18886656 chunk: [18886656, 18890752) 18890752, 4096 00020000:00200000:1.0:1789760732.728259:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760732.728279:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4611, 4611] 00000008:00010000:1.0:1789760732.728297:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760732.728303:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760732.728315:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18886656, 18890752) append 0 append lockpos 0 00000080:00200000:1.0:1789760732.728329:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18886656=0x1203000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760732.728427:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760732.728430:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18890752 00000020:00010000:1.0:1789760732.728438:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760732.728442:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760732.728445:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760732.728449:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760732.728451:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760732.728460:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760732.728467:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760732.728489:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760732.728662:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 18886656<>18890752 00000008:00010000:1.0:1789760732.808279:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760732.808294:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760732.808306:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760732.808329:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760732.808358:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507220352:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760733.388601:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4625, 4625] 00000080:00200000:1.0:1789760733.388619:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760733.388625:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760733.388640:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 18948096 i_blocks 38919 00000020:00200000:1.0:1789760733.388643:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760733.388654:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760733.388687:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 18948096, count: 4096 00000080:00200000:1.0:1789760733.388694:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 18948096, bytes: 4096 00000080:00200000:1.0:1789760733.388704:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760733.388713:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760733.388715:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [18948096, 18952192) 0 0 00000080:00200000:1.0:1789760733.388719:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760733.388736:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4626, 4626] 00020000:00200000:1.0:1789760733.388745:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 18948096 chunk: [18948096, 18952192) 18952192, 4096 00020000:00200000:1.0:1789760733.388749:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760733.388770:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4626, 4626] 00000008:00010000:1.0:1789760733.388791:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760733.388796:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760733.388808:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [18948096, 18952192) append 0 append lockpos 0 00000080:00200000:1.0:1789760733.388821:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=18948096=0x1212000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760733.388876:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760733.388878:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 18952192 00000020:00010000:1.0:1789760733.388887:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760733.388890:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760733.388893:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760733.388896:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760733.388898:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760733.388906:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760733.388913:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760733.388930:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760733.389034:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 18948096<>18952192 00000100:00100000:1.0:1789760733.389225:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760733.389246:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507222272:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760733.437061:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971946 gen 3 00000008:00010000:1.0:1789760733.437122:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760733.437131:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760733.437142:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760733.437203:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760733.437234:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507222272:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760733.441831:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507222400:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760733.516108:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971947 gen 3 00000008:00010000:1.0:1789760733.516162:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760733.516172:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760733.516185:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760733.516205:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760733.516238:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507222400:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760733.517178:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507222528:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760733.565077:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971948 gen 3 00000008:00010000:1.0:1789760733.565137:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760733.565145:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760733.565168:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760733.565192:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760733.565227:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507222528:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760733.566440:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760733.566465:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507222656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760733.645129:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971949 gen 3 00000080:00200000:1.0:1789760734.153204:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4640, 4640] 00000080:00200000:1.0:1789760734.153224:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760734.153231:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760734.153250:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19009536 i_blocks 38919 00000020:00200000:1.0:1789760734.153255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760734.153266:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760734.153302:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19009536, count: 4096 00000080:00200000:1.0:1789760734.153310:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19009536, bytes: 4096 00000080:00200000:1.0:1789760734.153320:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760734.153329:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760734.153332:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19009536, 19013632) 0 0 00000080:00200000:1.0:1789760734.153337:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760734.153364:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4641, 4641] 00020000:00200000:1.0:1789760734.153373:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19009536 chunk: [19009536, 19013632) 19013632, 4096 00020000:00200000:1.0:1789760734.153419:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760734.153441:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4641, 4641] 00000008:00010000:1.0:1789760734.153461:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760734.153467:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760734.153479:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19009536, 19013632) append 0 append lockpos 0 00000080:00200000:1.0:1789760734.153494:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19009536=0x1221000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760734.153778:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760734.153782:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19013632 00000020:00010000:1.0:1789760734.153791:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760734.153794:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760734.153798:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760734.153801:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760734.153804:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760734.153813:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760734.153819:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760734.153836:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760734.153944:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 19009536<>19013632 00000100:00100000:1.0:1789760734.202892:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971961 gen 3 00000008:00010000:1.0:1789760734.202948:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760734.202957:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.202968:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.202988:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760734.203020:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507224192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.270619:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971962 gen 3 00000008:00010000:1.0:1789760734.270685:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760734.270694:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.270707:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.270725:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760734.270761:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507224320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.329595:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971963 gen 3 00000008:00010000:1.0:1789760734.329649:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760734.329657:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.329670:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.329738:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760734.329770:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507224448:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.330720:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507224576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.362096:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971964 gen 3 00000008:00010000:1.0:1789760734.362147:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760734.362154:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.362166:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.362189:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760734.362222:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507224576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.362941:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507224704:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.403018:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971965 gen 3 00000008:00010000:1.0:1789760734.403087:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760734.403097:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.403111:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.403129:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760734.403160:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507224704:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.404080:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507224832:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.444267:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971966 gen 3 00000008:00010000:1.0:1789760734.444321:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760734.444329:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.444341:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.444447:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760734.444478:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507224832:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.446871:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507224960:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.498698:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971967 gen 3 00000008:00010000:1.0:1789760734.498748:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760734.498757:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.498837:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.498970:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760734.499003:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507224960:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.499863:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507225088:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.536091:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971968 gen 3 00000008:00010000:1.0:1789760734.536144:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760734.536152:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.536164:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.536185:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760734.536216:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507225088:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.537114:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225216:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.578288:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971969 gen 3 00000008:00010000:1.0:1789760734.578341:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760734.578349:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.578364:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.578565:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760734.578600:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225216:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.579261:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760734.579281:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225344:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.616567:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971970 gen 3 00000008:00010000:1.0:1789760734.616624:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760734.616634:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.616647:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.616673:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760734.616704:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225344:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.617601:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225472:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.649723:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971971 gen 3 00000008:00010000:1.0:1789760734.649775:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760734.649783:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.649795:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.649816:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760734.649843:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225472:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.650939:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760734.650960:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225600:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.688273:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971972 gen 3 00000008:00010000:1.0:1789760734.688320:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760734.688328:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.688341:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.688364:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760734.688438:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225600:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.689555:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225728:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.755279:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971973 gen 3 00000008:00010000:1.0:1789760734.755335:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760734.755343:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.755356:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.755489:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760734.755518:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507225728:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.763419:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507225856:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760734.820425:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760734.820441:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.820453:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.820617:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760734.820648:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507225856:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760734.859564:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4655, 4655] 00000080:00200000:1.0:1789760734.859581:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760734.859588:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760734.859604:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19070976 i_blocks 38919 00000020:00200000:1.0:1789760734.859608:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760734.859618:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760734.859655:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19070976, count: 4096 00000080:00200000:1.0:1789760734.859663:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19070976, bytes: 4096 00000080:00200000:1.0:1789760734.859673:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760734.859682:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760734.859684:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19070976, 19075072) 0 0 00000080:00200000:1.0:1789760734.859689:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760734.859707:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4656, 4656] 00020000:00200000:1.0:1789760734.859715:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19070976 chunk: [19070976, 19075072) 19075072, 4096 00020000:00200000:1.0:1789760734.859721:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760734.859740:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4656, 4656] 00000008:00010000:1.0:1789760734.859758:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760734.859764:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760734.859776:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19070976, 19075072) append 0 append lockpos 0 00000080:00200000:1.0:1789760734.859789:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19070976=0x1230000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760734.859841:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760734.859845:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19075072 00000020:00010000:1.0:1789760734.859854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760734.859857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760734.859861:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760734.859865:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760734.859868:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760734.859875:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760734.859882:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760734.859899:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760734.860001:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 19070976<>19075072 00000100:00100000:1.0:1789760734.860059:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760734.860144:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507226112:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.898971:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971976 gen 3 00000008:00010000:1.0:1789760734.899022:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760734.899032:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.899052:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.899078:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760734.899110:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507226112:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760734.934418:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760734.934436:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.934451:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.934472:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760734.934512:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507226240:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.935368:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507226368:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.960195:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971978 gen 3 00000008:00010000:1.0:1789760734.960266:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760734.960275:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.960287:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.960357:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760734.960421:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507226368:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.961665:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507226496:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.995564:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971979 gen 3 00000008:00010000:1.0:1789760734.995621:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760734.995630:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760734.995642:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760734.995664:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760734.995696:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507226496:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760734.996353:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507226624:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.030300:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971980 gen 3 00000008:00010000:1.0:1789760735.030353:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760735.030361:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.030373:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.030435:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760735.030469:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507226624:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760735.031141:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000008:00100000:1.0:1789760735.031260:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 19091456<>19095552 00000100:00100000:1.0:1789760735.031321:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507226752:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.065019:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971981 gen 3 00000008:00010000:1.0:1789760735.065085:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760735.065095:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.065108:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.065154:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760735.065192:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507226752:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.066168:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507226880:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.132450:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971982 gen 3 00000008:00010000:1.0:1789760735.132694:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760735.132703:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.132717:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.132778:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760735.132807:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507226880:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.133818:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507227008:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760735.209301:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760735.209317:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.209330:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.210756:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760735.210801:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507227008:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.213880:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227136:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.255813:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971983 gen 3 00000100:00100000:1.0:1789760735.255869:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b8a80 x1876699507227136/t4294971984(4294971984) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760751 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760735.255939:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760735.255947:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.255960:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.255980:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760735.256005:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227136:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.256792:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507227264:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.288442:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971985 gen 3 00000008:00010000:1.0:1789760735.288688:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760735.288699:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.288713:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.288735:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760735.288768:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507227264:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.289661:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227392:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.322691:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971986 gen 3 00000008:00010000:1.0:1789760735.322743:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760735.322752:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.322765:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.322787:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760735.322815:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227392:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.323636:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760735.323656:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227520:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.355952:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971987 gen 3 00000008:00010000:1.0:1789760735.356003:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760735.356011:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.356022:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.356051:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760735.356091:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227520:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.357231:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.399559:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971988 gen 3 00000008:00010000:1.0:1789760735.399609:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760735.399618:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.399630:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.399649:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760735.399680:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.400668:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760735.400692:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227776:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760735.439134:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760735.439154:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.439168:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.439190:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760735.439225:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507227776:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.440180:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760735.440200:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507227904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.481726:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971989 gen 3 00000100:00100000:1.0:1789760735.481772:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b8700 x1876699507227904/t4294971990(4294971990) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760751 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760735.481825:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760735.481834:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.481847:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.481868:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760735.481894:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507227904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.482788:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760735.482810:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507228032:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.514538:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294971991 gen 3 00000008:00010000:1.0:1789760735.514606:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760735.514614:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.514626:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.514647:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760735.514679:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507228032:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.515508:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760735.515526:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507228160:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760735.552657:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760735.552673:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760735.552687:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760735.552717:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760735.552748:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507228160:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.616116:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760735.616136:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507228544:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760735.995300:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507229568:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760736.527007:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760736.527044:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507231104:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760736.601728:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4695, 4695] 00000080:00200000:1.0:1789760736.601745:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760736.601751:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760736.601766:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19234816 i_blocks 38919 00000020:00200000:1.0:1789760736.601770:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760736.601779:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760736.601813:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19234816, count: 4096 00000080:00200000:1.0:1789760736.601821:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19234816, bytes: 4096 00000080:00200000:1.0:1789760736.601837:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760736.601847:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760736.601849:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19234816, 19238912) 0 0 00000080:00200000:1.0:1789760736.601853:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760736.601872:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4696, 4696] 00020000:00200000:1.0:1789760736.601881:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19234816 chunk: [19234816, 19238912) 19238912, 4096 00020000:00200000:1.0:1789760736.601886:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760736.601910:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4696, 4696] 00000008:00010000:1.0:1789760736.601931:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760736.601938:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760736.601964:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19234816, 19238912) append 0 append lockpos 0 00000080:00200000:1.0:1789760736.601978:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19234816=0x1258000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760736.602049:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760736.602052:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19238912 00000020:00010000:1.0:1789760736.602061:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760736.602064:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760736.602067:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760736.602072:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760736.602074:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760736.602082:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760736.602089:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760736.602107:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760736.602219:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 19234816<>19238912 00000100:00100000:1.0:1789760736.602264:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760736.602284:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507231232:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760736.671049:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4696, 4696] 00000080:00200000:1.0:1789760736.671084:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760736.671092:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760736.671111:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19238912 i_blocks 38919 00000020:00200000:1.0:1789760736.671116:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760736.671132:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760736.671169:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19238912, count: 4096 00000080:00200000:1.0:1789760736.671177:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19238912, bytes: 4096 00000080:00200000:1.0:1789760736.671192:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760736.671201:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760736.671203:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19238912, 19243008) 0 0 00000080:00200000:1.0:1789760736.671208:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760736.671225:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4697, 4697] 00020000:00200000:1.0:1789760736.671234:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19238912 chunk: [19238912, 19243008) 19243008, 4096 00020000:00200000:1.0:1789760736.671238:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760736.671263:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4697, 4697] 00000008:00010000:1.0:1789760736.671292:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760736.671818:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760736.671834:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19238912, 19243008) append 0 append lockpos 0 00000080:00200000:1.0:1789760736.671850:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19238912=0x1259000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760736.671922:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760736.671926:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19243008 00000020:00010000:1.0:1789760736.671945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760736.671949:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760736.671952:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760736.671956:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760736.671958:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760736.671967:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760736.671974:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760736.671995:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760736.672827:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19238912<>19243008 00000100:00100000:1.0:1789760736.708576:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972017 gen 3 00000008:00010000:1.0:1789760736.708906:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760736.708916:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760736.708928:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760736.709184:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760736.709216:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507231360:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760736.759803:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972018 gen 3 00000008:00010000:1.0:1789760736.759856:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760736.759864:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760736.759876:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760736.759905:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760736.759935:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507231488:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760736.807455:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972019 gen 3 00000008:00010000:1.0:1789760736.807506:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760736.807515:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760736.807529:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760736.807554:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760736.807586:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507231616:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760736.911129:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4701, 4701] 00000080:00200000:1.0:1789760736.911153:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760736.911160:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760736.911176:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19259392 i_blocks 38919 00000020:00200000:1.0:1789760736.911181:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760736.911196:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760736.911231:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19259392, count: 4096 00000080:00200000:1.0:1789760736.911239:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19259392, bytes: 4096 00000080:00200000:1.0:1789760736.911252:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760736.911269:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760736.911271:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19259392, 19263488) 0 0 00000080:00200000:1.0:1789760736.911276:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760736.911292:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4702, 4702] 00020000:00200000:1.0:1789760736.911300:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19259392 chunk: [19259392, 19263488) 19263488, 4096 00020000:00200000:1.0:1789760736.911304:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760736.911325:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4702, 4702] 00000008:00010000:1.0:1789760736.911342:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760736.911348:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760736.911360:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19259392, 19263488) append 0 append lockpos 0 00000080:00200000:1.0:1789760736.911372:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19259392=0x125e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760736.911468:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760736.911471:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19263488 00000020:00010000:1.0:1789760736.911480:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760736.911483:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760736.911487:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760736.911490:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760736.911493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760736.911501:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760736.911507:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760736.911528:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760736.911651:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19259392<>19263488 00000100:00100000:1.0:1789760736.911710:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760736.911739:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507232000:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760736.945979:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972022 gen 3 00000008:00010000:1.0:1789760736.946118:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760736.946126:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760736.946137:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760736.946204:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760736.946237:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507232000:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760736.948227:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507232128:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760736.988802:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972023 gen 3 00000008:00010000:1.0:1789760736.988853:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760736.988861:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760736.988874:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760736.988946:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760736.988977:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507232128:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.019846:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2416:1876699507232384:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760737.021186:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760737.021209:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507232512:192.168.203.141@tcp:400:kworker.0 00000080:00200000:1.0:1789760737.027529:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4704, 4704] 00000080:00200000:1.0:1789760737.027548:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760737.027555:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760737.027572:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19271680 i_blocks 38919 00000020:00200000:1.0:1789760737.027575:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760737.027585:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760737.027615:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19271680, count: 4096 00000080:00200000:1.0:1789760737.027622:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19271680, bytes: 4096 00000080:00200000:1.0:1789760737.027636:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760737.027644:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760737.027647:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19271680, 19275776) 0 0 00000080:00200000:1.0:1789760737.027652:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760737.027679:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4705, 4705] 00020000:00200000:1.0:1789760737.027688:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19271680 chunk: [19271680, 19275776) 19275776, 4096 00020000:00200000:1.0:1789760737.027692:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760737.027711:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4705, 4705] 00000008:00010000:1.0:1789760737.027729:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760737.027735:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760737.027746:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19271680, 19275776) append 0 append lockpos 0 00000080:00200000:1.0:1789760737.027763:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19271680=0x1261000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760737.027819:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760737.027822:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19275776 00000020:00010000:1.0:1789760737.027830:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760737.027834:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760737.027837:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760737.027840:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760737.027843:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760737.027850:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760737.027856:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760737.027871:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760737.027978:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19271680<>19275776 00000100:00100000:1.0:1789760737.028012:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760737.028041:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507232640:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.031072:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2416:1876699507232384:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760737.040935:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507232512:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760737.068723:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972025 gen 3 00000008:00010000:1.0:1789760737.069028:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760737.069037:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.069048:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.069144:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760737.069175:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507232640:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760737.190833:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4707, 4707] 00000080:00200000:1.0:1789760737.190859:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760737.190865:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760737.190880:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19283968 i_blocks 38919 00000020:00200000:1.0:1789760737.190883:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760737.190893:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760737.190929:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19283968, count: 4096 00000080:00200000:1.0:1789760737.190936:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19283968, bytes: 4096 00000080:00200000:1.0:1789760737.190945:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760737.190954:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760737.190957:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19283968, 19288064) 0 0 00000080:00200000:1.0:1789760737.190961:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760737.190977:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4708, 4708] 00020000:00200000:1.0:1789760737.190985:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19283968 chunk: [19283968, 19288064) 19288064, 4096 00020000:00200000:1.0:1789760737.190989:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760737.191008:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4708, 4708] 00000008:00010000:1.0:1789760737.191037:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760737.191044:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760737.191056:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19283968, 19288064) append 0 append lockpos 0 00000080:00200000:1.0:1789760737.191071:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19283968=0x1264000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760737.191129:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760737.191132:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19288064 00000020:00010000:1.0:1789760737.191141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760737.191144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760737.191148:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760737.191151:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760737.191154:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760737.191163:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760737.191169:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760737.191186:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760737.191293:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19283968<>19288064 00000100:00100000:1.0:1789760737.191344:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760737.191360:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507233408:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.279886:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972028 gen 3 00000008:00010000:1.0:1789760737.279951:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760737.279957:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.279966:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.279980:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760737.280003:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507233408:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.281094:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507233536:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760737.328733:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760737.328751:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.328765:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.328847:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760737.328880:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507233536:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.331213:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507233664:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.360872:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972030 gen 3 00000008:00010000:1.0:1789760737.360937:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760737.360946:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.360958:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.360977:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760737.361005:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507233664:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.361784:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760737.361815:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507233792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.405188:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972031 gen 3 00000008:00010000:1.0:1789760737.405250:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760737.405259:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.405272:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.405341:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760737.405433:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507233792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.407173:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507233920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.439827:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972032 gen 3 00000008:00010000:1.0:1789760737.439878:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760737.439887:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.439898:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.439919:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760737.439951:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507233920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.440639:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507234048:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760737.484368:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760737.484422:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.484436:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.491271:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760737.491304:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507234048:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760737.550306:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972034 gen 3 00000008:00010000:1.0:1789760737.551037:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760737.551047:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760737.551261:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760737.551352:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760737.551408:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507234176:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.073233:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760738.073258:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507235456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.100938:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972044 gen 3 00000008:00010000:1.0:1789760738.100994:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760738.101002:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.101023:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.101041:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760738.101071:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507235456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.101814:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760738.101835:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507235584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.138065:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972045 gen 3 00000008:00010000:1.0:1789760738.138114:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760738.138123:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.138135:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.138158:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760738.138190:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507235584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.167890:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972046 gen 3 00000008:00010000:1.0:1789760738.167941:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760738.167950:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.167963:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.168036:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760738.168067:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507235712:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.168912:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507235840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.197098:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972047 gen 3 00000008:00010000:1.0:1789760738.197149:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760738.197158:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.197170:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.197188:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760738.197214:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507235840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.197988:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507235968:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.236809:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972048 gen 3 00000008:00010000:1.0:1789760738.236864:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760738.236873:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.236887:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.236967:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760738.237004:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507235968:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.238847:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507236096:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.321981:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972049 gen 3 00000008:00010000:1.0:1789760738.322043:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760738.322052:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.322066:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.322089:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760738.322122:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507236096:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.623765:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237120:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.657627:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972057 gen 3 00000008:00010000:1.0:1789760738.657677:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760738.657687:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.658132:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.658158:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760738.658195:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237120:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.661168:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507237248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.724605:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972058 gen 3 00000008:00010000:1.0:1789760738.724656:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760738.724665:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.724679:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.724702:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760738.724736:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507237248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.728373:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237376:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.785869:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972059 gen 3 00000008:00010000:1.0:1789760738.785973:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760738.785982:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.785996:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.787201:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760738.787249:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237376:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.791034:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507237504:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760738.823265:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760738.823281:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.823294:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.823369:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760738.823443:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507237504:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.825089:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237632:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.867662:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972061 gen 3 00000008:00010000:1.0:1789760738.867738:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760738.867746:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.867760:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.867848:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760738.867887:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237632:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760738.934863:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4742, 4742] 00000080:00200000:1.0:1789760738.934881:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760738.934888:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760738.934903:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19427328 i_blocks 38919 00000020:00200000:1.0:1789760738.934907:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760738.934917:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760738.934951:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19427328, count: 4096 00000080:00200000:1.0:1789760738.934958:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19427328, bytes: 4096 00000080:00200000:1.0:1789760738.934969:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760738.934978:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760738.934980:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19427328, 19431424) 0 0 00000080:00200000:1.0:1789760738.934985:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760738.935001:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4743, 4743] 00020000:00200000:1.0:1789760738.935020:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19427328 chunk: [19427328, 19431424) 19431424, 4096 00020000:00200000:1.0:1789760738.935025:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760738.935044:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4743, 4743] 00000008:00010000:1.0:1789760738.935061:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760738.935066:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760738.935077:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19427328, 19431424) append 0 append lockpos 0 00000080:00200000:1.0:1789760738.935090:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19427328=0x1287000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760738.935143:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760738.935146:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19431424 00000020:00010000:1.0:1789760738.935154:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760738.935157:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760738.935160:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760738.935164:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760738.935166:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760738.935173:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760738.935180:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760738.935196:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760738.935299:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19427328<>19431424 00000100:00100000:1.0:1789760738.935347:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760738.935364:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237888:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760738.977235:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972063 gen 3 00000008:00010000:1.0:1789760738.977275:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760738.977281:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760738.977290:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760738.977317:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760738.977340:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507237888:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760739.009625:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4744, 4744] 00000080:00200000:1.0:1789760739.009637:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760739.009642:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760739.009655:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19435520 i_blocks 38919 00000020:00200000:1.0:1789760739.009659:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760739.009667:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760739.009771:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19435520, count: 4096 00000080:00200000:1.0:1789760739.009778:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19435520, bytes: 4096 00000080:00200000:1.0:1789760739.009787:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760739.009794:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760739.009796:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19435520, 19439616) 0 0 00000080:00200000:1.0:1789760739.009800:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760739.009814:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4745, 4745] 00020000:00200000:1.0:1789760739.009822:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19435520 chunk: [19435520, 19439616) 19439616, 4096 00020000:00200000:1.0:1789760739.009826:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760739.009854:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4745, 4745] 00000008:00010000:1.0:1789760739.009868:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760739.009872:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760739.010160:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19435520, 19439616) append 0 append lockpos 0 00000080:00200000:1.0:1789760739.010237:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19435520=0x1289000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760739.010287:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760739.010290:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19439616 00000020:00010000:1.0:1789760739.010296:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760739.010299:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.010302:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.010306:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760739.010309:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760739.010315:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760739.010321:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760739.010338:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760739.010458:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19435520<>19439616 00000100:00100000:1.0:1789760739.045240:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972064 gen 3 00000100:00100000:1.0:1789760739.045271:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8168000 x1876699507238144/t4294972065(4294972065) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760755 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760739.045309:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760739.045314:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760739.045321:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760739.045345:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760739.045364:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507238144:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760739.046452:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:1.0:1789760739.046532:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 19439616<>19443712 00000100:00100000:1.0:1789760739.046577:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507238272:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760739.088529:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972066 gen 3 00000008:00010000:1.0:1789760739.088576:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50acc00 00000020:00200000:1.0:1789760739.088584:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760739.088592:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760739.088621:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50acc00 00000100:00100000:1.0:1789760739.088648:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507238272:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760739.139249:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507238528:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760739.189783:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972068 gen 3 00000008:00010000:1.0:1789760739.189833:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760739.189843:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760739.189856:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760739.189879:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760739.189919:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507238528:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760739.191169:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760739.191195:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507238656:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760739.259625:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760739.259641:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760739.259654:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760739.259741:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760739.259776:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507238656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760739.294765:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972070 gen 3 00000008:00010000:1.0:1789760739.294830:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760739.294839:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760739.294851:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760739.294873:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760739.294913:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507238784:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760739.442171:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4752, 4752] 00000080:00200000:1.0:1789760739.442185:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760739.442190:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760739.442202:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19468288 i_blocks 38919 00000020:00200000:1.0:1789760739.442205:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760739.442214:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760739.442239:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19468288, count: 4096 00000080:00200000:1.0:1789760739.442245:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19468288, bytes: 4096 00000080:00200000:1.0:1789760739.442252:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760739.442265:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760739.442267:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19468288, 19472384) 0 0 00000080:00200000:1.0:1789760739.442271:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760739.442284:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4753, 4753] 00020000:00200000:1.0:1789760739.442290:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19468288 chunk: [19468288, 19472384) 19472384, 4096 00020000:00200000:1.0:1789760739.442294:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760739.442309:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4753, 4753] 00000008:00010000:1.0:1789760739.442323:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760739.442327:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760739.442337:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19468288, 19472384) append 0 append lockpos 0 00000080:00200000:1.0:1789760739.442348:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19468288=0x1291000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760739.442524:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760739.442527:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19472384 00000020:00010000:1.0:1789760739.442534:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760739.442538:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.442541:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.442544:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760739.442546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760739.442553:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760739.442560:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760739.442574:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760739.442648:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 19468288<>19472384 00000008:00010000:1.0:1789760739.507140:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760739.507151:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760739.507160:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760739.507177:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760739.507200:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507239168:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760739.877578:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4760, 4760] 00000080:00200000:1.0:1789760739.877600:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760739.877607:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760739.877624:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19501056 i_blocks 38919 00000020:00200000:1.0:1789760739.877630:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760739.877644:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760739.877677:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19501056, count: 4096 00000080:00200000:1.0:1789760739.877686:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19501056, bytes: 4096 00000080:00200000:1.0:1789760739.877701:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760739.877711:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760739.877713:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19501056, 19505152) 0 0 00000080:00200000:1.0:1789760739.877719:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760739.877736:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4761, 4761] 00020000:00200000:1.0:1789760739.877745:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19501056 chunk: [19501056, 19505152) 19505152, 4096 00020000:00200000:1.0:1789760739.877750:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760739.877772:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4761, 4761] 00000008:00010000:1.0:1789760739.877793:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760739.877800:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760739.877813:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19501056, 19505152) append 0 append lockpos 0 00000080:00200000:1.0:1789760739.877830:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19501056=0x1299000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760739.878106:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760739.878109:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19505152 00000020:00010000:1.0:1789760739.878120:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760739.878124:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.878127:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.878131:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760739.878134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760739.878143:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760739.878150:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760739.878172:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760739.878280:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19501056<>19505152 00000100:00100000:1.0:1789760739.934233:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972081 gen 3 00000008:00010000:1.0:1789760739.934286:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760739.934294:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760739.934305:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760739.934340:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760739.934416:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507240192:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760739.971647:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4762, 4762] 00000080:00200000:1.0:1789760739.971663:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760739.971669:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760739.971686:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19509248 i_blocks 38919 00000020:00200000:1.0:1789760739.971689:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760739.971700:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760739.971733:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19509248, count: 4096 00000080:00200000:1.0:1789760739.971741:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19509248, bytes: 4096 00000080:00200000:1.0:1789760739.971753:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760739.971761:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760739.971764:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19509248, 19513344) 0 0 00000080:00200000:1.0:1789760739.971768:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760739.971784:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4763, 4763] 00020000:00200000:1.0:1789760739.971793:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19509248 chunk: [19509248, 19513344) 19513344, 4096 00020000:00200000:1.0:1789760739.971798:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760739.971818:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4763, 4763] 00000008:00010000:1.0:1789760739.971837:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760739.971843:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760739.971854:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19509248, 19513344) append 0 append lockpos 0 00000080:00200000:1.0:1789760739.971868:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19509248=0x129b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760739.972607:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760739.972613:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19513344 00000020:00010000:1.0:1789760739.972624:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760739.972628:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.972632:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760739.972636:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760739.972639:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760739.972649:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760739.972656:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760739.972678:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760739.972793:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19509248<>19513344 00000100:00100000:1.0:1789760740.014021:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972083 gen 3 00000008:00010000:1.0:1789760740.014075:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760740.014084:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.014096:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.014121:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760740.014153:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507240448:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760740.106724:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4765, 4765] 00000080:00200000:1.0:1789760740.106739:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760740.106744:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760740.106755:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19521536 i_blocks 38919 00000020:00200000:1.0:1789760740.106759:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760740.106767:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760740.106792:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19521536, count: 4096 00000080:00200000:1.0:1789760740.106796:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19521536, bytes: 4096 00000080:00200000:1.0:1789760740.106802:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760740.106807:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760740.106808:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19521536, 19525632) 0 0 00000080:00200000:1.0:1789760740.106812:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760740.106821:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4766, 4766] 00020000:00200000:1.0:1789760740.106826:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19521536 chunk: [19521536, 19525632) 19525632, 4096 00020000:00200000:1.0:1789760740.106829:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760740.106841:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4766, 4766] 00000008:00010000:1.0:1789760740.106852:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760740.106856:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760740.106863:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19521536, 19525632) append 0 append lockpos 0 00000080:00200000:1.0:1789760740.106872:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19521536=0x129e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760740.106921:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760740.106924:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19525632 00000020:00010000:1.0:1789760740.106930:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760740.106933:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.106936:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.106939:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760740.106942:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760740.106949:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760740.106954:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760740.106967:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760740.107074:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 19521536<>19525632 00000100:00100000:1.0:1789760740.107112:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760740.107126:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507240832:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760740.142120:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760740.142131:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.142140:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.143054:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760740.143086:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507240832:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760740.408229:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4772, 4772] 00000080:00200000:1.0:1789760740.408403:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760740.408410:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760740.408422:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19550208 i_blocks 38919 00000020:00200000:1.0:1789760740.408426:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760740.408436:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760740.408459:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19550208, count: 4096 00000080:00200000:1.0:1789760740.408466:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19550208, bytes: 4096 00000080:00200000:1.0:1789760740.408474:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760740.408482:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760740.408484:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19550208, 19554304) 0 0 00000080:00200000:1.0:1789760740.408488:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760740.408499:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4773, 4773] 00020000:00200000:1.0:1789760740.408506:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19550208 chunk: [19550208, 19554304) 19554304, 4096 00020000:00200000:1.0:1789760740.409443:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760740.409464:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4773, 4773] 00000008:00010000:1.0:1789760740.409479:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760740.409484:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760740.409494:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19550208, 19554304) append 0 append lockpos 0 00000080:00200000:1.0:1789760740.409505:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19550208=0x12a5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760740.409565:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760740.409569:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19554304 00000020:00010000:1.0:1789760740.409575:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760740.409578:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.409582:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.409585:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760740.409588:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760740.409594:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760740.409600:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760740.409617:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760740.409697:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 19550208<>19554304 00000100:00100000:1.0:1789760740.409745:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760740.409772:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507241728:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.448265:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972093 gen 3 00000008:00010000:1.0:1789760740.448304:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760740.448311:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.448319:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.448340:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760740.448362:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507241728:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760740.497691:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4774, 4774] 00000080:00200000:1.0:1789760740.497704:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760740.497710:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760740.497722:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19558400 i_blocks 38919 00000020:00200000:1.0:1789760740.497726:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760740.497735:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760740.497761:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19558400, count: 4096 00000080:00200000:1.0:1789760740.497768:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19558400, bytes: 4096 00000080:00200000:1.0:1789760740.497777:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760740.497785:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760740.497787:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19558400, 19562496) 0 0 00000080:00200000:1.0:1789760740.497791:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760740.497804:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4775, 4775] 00020000:00200000:1.0:1789760740.497810:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19558400 chunk: [19558400, 19562496) 19562496, 4096 00020000:00200000:1.0:1789760740.497814:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760740.497828:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4775, 4775] 00000008:00010000:1.0:1789760740.497841:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760740.497845:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760740.497853:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19558400, 19562496) append 0 append lockpos 0 00000080:00200000:1.0:1789760740.497863:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19558400=0x12a7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760740.497902:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760740.497904:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19562496 00000020:00010000:1.0:1789760740.497910:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760740.497913:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.497916:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.497919:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760740.497921:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760740.497927:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760740.497932:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760740.497945:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760740.498028:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 19558400<>19562496 00000008:00010000:1.0:1789760740.536951:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760740.536962:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.536970:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.536989:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760740.537021:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507241984:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760740.626874:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4777, 4777] 00000080:00200000:1.0:1789760740.626888:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760740.626893:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760740.626905:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19570688 i_blocks 38919 00000020:00200000:1.0:1789760740.626908:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760740.626916:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760740.626942:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19570688, count: 4096 00000080:00200000:1.0:1789760740.626948:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19570688, bytes: 4096 00000080:00200000:1.0:1789760740.626956:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760740.626963:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760740.626965:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19570688, 19574784) 0 0 00000080:00200000:1.0:1789760740.626970:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760740.626983:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4778, 4778] 00020000:00200000:1.0:1789760740.626999:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19570688 chunk: [19570688, 19574784) 19574784, 4096 00020000:00200000:1.0:1789760740.627003:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760740.627019:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4778, 4778] 00000008:00010000:1.0:1789760740.627043:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760740.627048:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760740.627058:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19570688, 19574784) append 0 append lockpos 0 00000080:00200000:1.0:1789760740.627068:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19570688=0x12aa000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760740.627112:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760740.627115:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19574784 00000020:00010000:1.0:1789760740.627123:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760740.627127:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.627129:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760740.627133:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760740.627135:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760740.627142:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760740.627147:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760740.627161:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760740.627239:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 19570688<>19574784 00000100:00100000:1.0:1789760740.627279:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760740.627292:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507242368:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.678241:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972098 gen 3 00000008:00010000:1.0:1789760740.678287:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760740.678293:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.678300:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.678323:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760740.678345:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507242368:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.719336:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972099 gen 3 00000008:00010000:1.0:1789760740.719371:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760740.719394:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.719403:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.719420:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760740.719442:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507242496:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.724346:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760740.724368:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507242624:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.774590:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972100 gen 3 00000008:00010000:1.0:1789760740.774628:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760740.774635:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.774645:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.774724:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760740.774750:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507242624:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.777046:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507242752:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.818091:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972101 gen 3 00000008:00010000:1.0:1789760740.818128:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760740.818135:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.818144:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.818161:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760740.818183:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507242752:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.819166:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507242880:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.856361:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972102 gen 3 00000008:00010000:1.0:1789760740.856429:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760740.856501:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.856512:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.856531:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760740.856554:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507242880:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.857425:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243008:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.895086:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972103 gen 3 00000008:00010000:1.0:1789760740.895123:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760740.895129:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.895138:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.895157:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760740.895179:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243008:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.895851:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507243136:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.934507:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972104 gen 3 00000008:00010000:1.0:1789760740.934545:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760740.934551:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.934562:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.934581:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760740.934608:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507243136:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.935273:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243264:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760740.973480:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760740.973492:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760740.973503:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760740.973521:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760740.973544:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243264:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760740.974250:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507243392:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.021970:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972106 gen 3 00000008:00010000:1.0:1789760741.022034:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760741.022043:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.022055:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.022076:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760741.022105:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507243392:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.023226:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243520:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.071699:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972107 gen 3 00000008:00010000:1.0:1789760741.071735:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760741.071741:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.071751:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.071809:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760741.071834:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243520:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.072604:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507243648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.137205:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972108 gen 3 00000008:00010000:1.0:1789760741.137244:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760741.137251:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.137262:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.137278:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760741.137301:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507243648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.138937:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243776:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760741.177350:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760741.177362:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.177372:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.177424:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760741.177462:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243776:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.178815:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760741.178833:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.228996:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972110 gen 3 00000008:00010000:1.0:1789760741.229190:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760741.229197:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.229205:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.229264:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760741.229287:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507243904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.232809:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507244032:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.287643:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972111 gen 3 00000008:00010000:1.0:1789760741.287679:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760741.287685:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.287694:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.287712:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760741.287734:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507244032:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.288410:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507244160:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760741.353465:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760741.353477:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.353487:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.353509:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760741.353534:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507244160:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.354686:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507244288:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.401275:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972112 gen 3 00000100:00100000:1.0:1789760741.401307:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25bb480 x1876699507244288/t4294972113(4294972113) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760757 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760741.401341:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760741.401346:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.401354:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.401434:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760741.401454:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507244288:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.403038:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507244416:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.452428:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972114 gen 3 00000008:00010000:1.0:1789760741.452549:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760741.452556:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.452564:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.452639:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760741.452667:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507244416:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.453891:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760741.453914:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507244544:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.512965:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972115 gen 3 00000008:00010000:1.0:1789760741.513012:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760741.513019:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.513029:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.513104:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760741.513129:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507244544:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760741.568956:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4796, 4796] 00000080:00200000:1.0:1789760741.568969:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760741.568974:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760741.568995:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19648512 i_blocks 38919 00000020:00200000:1.0:1789760741.568998:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760741.569007:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760741.569031:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19648512, count: 4096 00000080:00200000:1.0:1789760741.569037:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19648512, bytes: 4096 00000080:00200000:1.0:1789760741.569045:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760741.569053:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760741.569056:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19648512, 19652608) 0 0 00000080:00200000:1.0:1789760741.569059:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760741.569072:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4797, 4797] 00020000:00200000:1.0:1789760741.569078:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19648512 chunk: [19648512, 19652608) 19652608, 4096 00020000:00200000:1.0:1789760741.569083:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760741.569099:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4797, 4797] 00000008:00010000:1.0:1789760741.569112:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760741.569116:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760741.569124:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19648512, 19652608) append 0 append lockpos 0 00000080:00200000:1.0:1789760741.569134:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19648512=0x12bd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760741.569179:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760741.569182:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19652608 00000020:00010000:1.0:1789760741.569188:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760741.569191:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760741.569194:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760741.569197:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760741.569199:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760741.569205:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760741.569301:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760741.569317:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760741.569720:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 19648512<>19652608 00000100:00100000:1.0:1789760741.607323:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972117 gen 3 00000008:00010000:1.0:1789760741.607361:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760741.607367:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.607408:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.607638:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760741.607662:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507244800:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760741.672122:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972118 gen 3 00000008:00010000:1.0:1789760741.672157:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760741.672163:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.672172:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.672197:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760741.672221:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507244928:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760741.799098:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4801, 4801] 00000080:00200000:1.0:1789760741.799111:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760741.799116:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760741.799128:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19668992 i_blocks 38919 00000020:00200000:1.0:1789760741.799132:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760741.799141:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760741.799168:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19668992, count: 4096 00000080:00200000:1.0:1789760741.799175:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19668992, bytes: 4096 00000080:00200000:1.0:1789760741.799184:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760741.799192:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760741.799194:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19668992, 19673088) 0 0 00000080:00200000:1.0:1789760741.799198:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760741.799213:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4802, 4802] 00020000:00200000:1.0:1789760741.799219:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19668992 chunk: [19668992, 19673088) 19673088, 4096 00020000:00200000:1.0:1789760741.799223:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760741.799238:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4802, 4802] 00000008:00010000:1.0:1789760741.799252:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760741.799257:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760741.799266:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19668992, 19673088) append 0 append lockpos 0 00000080:00200000:1.0:1789760741.799276:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19668992=0x12c2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760741.799318:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760741.799321:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19673088 00000020:00010000:1.0:1789760741.799327:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760741.799330:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760741.799333:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760741.799336:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760741.799338:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760741.799343:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760741.799349:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760741.799362:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760741.799472:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 19668992<>19673088 00000100:00100000:1.0:1789760741.799501:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760741.799514:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507245440:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760741.868080:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4802, 4802] 00000080:00200000:1.0:1789760741.868093:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760741.868098:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760741.868111:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19673088 i_blocks 38919 00000020:00200000:1.0:1789760741.868114:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760741.868122:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760741.868147:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19673088, count: 4096 00000080:00200000:1.0:1789760741.868153:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19673088, bytes: 4096 00000080:00200000:1.0:1789760741.868161:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760741.868168:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760741.868170:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19673088, 19677184) 0 0 00000080:00200000:1.0:1789760741.868174:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760741.868187:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4803, 4803] 00020000:00200000:1.0:1789760741.868193:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19673088 chunk: [19673088, 19677184) 19677184, 4096 00020000:00200000:1.0:1789760741.868197:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760741.868211:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4803, 4803] 00000008:00010000:1.0:1789760741.868224:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760741.868229:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760741.868238:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19673088, 19677184) append 0 append lockpos 0 00000080:00200000:1.0:1789760741.868248:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19673088=0x12c3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760741.868290:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760741.868293:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19677184 00000020:00010000:1.0:1789760741.868299:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760741.868303:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760741.868306:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760741.868308:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760741.868311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760741.868318:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760741.868323:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760741.868338:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760741.868509:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 19673088<>19677184 00000100:00100000:1.0:1789760741.916565:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972123 gen 3 00000008:00010000:1.0:1789760741.916615:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760741.916621:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760741.916628:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760741.916693:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760741.916714:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507245568:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760742.520150:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4815, 4815] 00000080:00200000:1.0:1789760742.520173:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760742.520179:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760742.520190:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19726336 i_blocks 38919 00000020:00200000:1.0:1789760742.520193:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760742.520201:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760742.520225:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19726336, count: 4096 00000080:00200000:1.0:1789760742.520231:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19726336, bytes: 4096 00000080:00200000:1.0:1789760742.520239:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760742.520247:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760742.520249:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19726336, 19730432) 0 0 00000080:00200000:1.0:1789760742.520252:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760742.520264:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4816, 4816] 00020000:00200000:1.0:1789760742.520270:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19726336 chunk: [19726336, 19730432) 19730432, 4096 00020000:00200000:1.0:1789760742.520274:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760742.520287:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4816, 4816] 00000008:00010000:1.0:1789760742.520300:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760742.520304:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760742.520313:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19726336, 19730432) append 0 append lockpos 0 00000080:00200000:1.0:1789760742.520325:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19726336=0x12d0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760742.520369:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760742.520372:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19730432 00000020:00010000:1.0:1789760742.520411:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760742.520415:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760742.520417:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760742.520421:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760742.520424:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760742.520430:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760742.520436:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760742.520448:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760742.520691:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 19726336<>19730432 00000100:00100000:1.0:1789760742.587654:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972136 gen 3 00000008:00010000:1.0:1789760742.587701:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760742.587707:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760742.587716:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760742.587743:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760742.587766:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507247872:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760742.633696:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4817, 4817] 00000080:00200000:1.0:1789760742.633712:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760742.633717:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760742.633730:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19734528 i_blocks 38919 00000020:00200000:1.0:1789760742.633733:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760742.633742:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760742.633770:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19734528, count: 4096 00000080:00200000:1.0:1789760742.633776:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19734528, bytes: 4096 00000080:00200000:1.0:1789760742.633784:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760742.633792:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760742.633794:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19734528, 19738624) 0 0 00000080:00200000:1.0:1789760742.633798:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760742.633810:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4818, 4818] 00020000:00200000:1.0:1789760742.633816:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19734528 chunk: [19734528, 19738624) 19738624, 4096 00020000:00200000:1.0:1789760742.633820:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760742.633835:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4818, 4818] 00000008:00010000:1.0:1789760742.633849:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760742.633854:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760742.633863:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19734528, 19738624) append 0 append lockpos 0 00000080:00200000:1.0:1789760742.633880:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19734528=0x12d2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760742.633925:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760742.633928:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19738624 00000020:00010000:1.0:1789760742.633935:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760742.633939:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760742.633941:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54171b8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760742.633945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760742.633948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760742.633954:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760742.633959:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760742.633986:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760742.634133:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 19734528<>19738624 00000100:00100000:1.0:1789760742.675006:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972138 gen 3 00000008:00010000:1.0:1789760742.675041:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760742.675048:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760742.675057:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760742.675166:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760742.675185:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507248128:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760742.905238:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507248896:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760742.947861:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760742.947873:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760742.947882:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760742.947999:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760742.948024:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507248896:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760742.949437:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507249024:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760742.985448:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972145 gen 3 00000008:00010000:1.0:1789760742.985499:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760742.985506:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760742.985515:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760742.985660:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760742.985685:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507249024:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760742.988141:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507249152:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.034361:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972146 gen 3 00000008:00010000:1.0:1789760743.034411:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760743.034418:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.034537:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.034626:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760743.034650:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507249152:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.038605:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507249280:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.082256:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972147 gen 3 00000008:00010000:1.0:1789760743.082292:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760743.082298:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.082308:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.082494:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760743.082519:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507249280:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760743.083197:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19775488 i_blocks 38919 00000020:00200000:1.0:1789760743.083203:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760743.083213:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760743.083240:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19775488, count: 4096 00000080:00200000:1.0:1789760743.083247:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19775488, bytes: 4096 00000080:00200000:1.0:1789760743.083256:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760743.083263:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760743.083266:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19775488, 19779584) 0 0 00000080:00200000:1.0:1789760743.083270:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760743.083285:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4828, 4828] 00020000:00200000:1.0:1789760743.083292:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19775488 chunk: [19775488, 19779584) 19779584, 4096 00020000:00200000:1.0:1789760743.083296:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760743.083311:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4828, 4828] 00000008:00010000:1.0:1789760743.083325:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760743.083330:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760743.083339:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19775488, 19779584) append 0 append lockpos 0 00000080:00200000:1.0:1789760743.083350:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19775488=0x12dc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760743.083485:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760743.083488:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19779584 00000020:00010000:1.0:1789760743.083497:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760743.083500:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760743.083503:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5c8b3f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760743.083507:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760743.083510:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760743.083518:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760743.083524:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760743.083538:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760743.083616:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 19775488<>19779584 00000100:00100000:1.0:1789760743.128804:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972148 gen 3 00000008:00010000:1.0:1789760743.128842:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760743.128848:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.128859:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.128885:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760743.128919:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507249408:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760743.178875:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4829, 4829] 00000080:00200000:1.0:1789760743.178891:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760743.178896:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760743.178909:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19783680 i_blocks 38919 00000020:00200000:1.0:1789760743.178913:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760743.178923:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760743.178952:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19783680, count: 4096 00000080:00200000:1.0:1789760743.178958:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19783680, bytes: 4096 00000080:00200000:1.0:1789760743.178966:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760743.178984:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760743.178987:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19783680, 19787776) 0 0 00000080:00200000:1.0:1789760743.178991:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760743.179005:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4830, 4830] 00020000:00200000:1.0:1789760743.179012:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19783680 chunk: [19783680, 19787776) 19787776, 4096 00020000:00200000:1.0:1789760743.179016:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760743.179577:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4830, 4830] 00000008:00010000:1.0:1789760743.179594:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760743.179599:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760743.179608:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19783680, 19787776) append 0 append lockpos 0 00000080:00200000:1.0:1789760743.179619:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19783680=0x12de000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760743.179668:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760743.179671:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19787776 00000020:00010000:1.0:1789760743.179677:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760743.179680:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760743.179691:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5c8b3f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760743.179694:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760743.179696:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760743.179703:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760743.179709:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760743.179725:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760743.179802:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 19783680<>19787776 00000100:00100000:1.0:1789760743.179841:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760743.179856:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507249664:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.223899:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972150 gen 3 00000008:00010000:1.0:1789760743.223938:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760743.223945:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.223954:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.223986:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760743.224009:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507249664:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.224805:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507249792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.265269:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972151 gen 3 00000008:00010000:1.0:1789760743.265312:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760743.265321:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.265332:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.265411:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760743.265444:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507249792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.267128:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507249920:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760743.307818:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760743.307830:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.307841:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.307925:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760743.307952:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507249920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.308786:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507250048:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.372314:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972153 gen 3 00000008:00010000:1.0:1789760743.372411:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760743.372418:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.372427:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.372494:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760743.372519:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507250048:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.373328:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507250176:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.417103:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972154 gen 3 00000008:00010000:1.0:1789760743.417140:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760743.417147:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.417157:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.417238:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760743.417265:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507250176:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.417859:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507250304:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.469531:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972155 gen 3 00000008:00010000:1.0:1789760743.469565:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760743.469583:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.469593:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.469610:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760743.469642:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507250304:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.470581:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507250432:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760743.515150:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760743.515161:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.515171:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.515238:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760743.515262:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507250432:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.519323:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507250560:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.558872:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972157 gen 3 00000008:00010000:1.0:1789760743.558916:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760743.558922:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.558931:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.559007:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760743.559031:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507250560:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760743.819014:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4842, 4842] 00000080:00200000:1.0:1789760743.819029:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760743.819035:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760743.819047:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 19836928 i_blocks 38919 00000020:00200000:1.0:1789760743.819051:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760743.819059:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760743.819085:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 19836928, count: 4096 00000080:00200000:1.0:1789760743.819092:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 19836928, bytes: 4096 00000080:00200000:1.0:1789760743.819101:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760743.819109:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760743.819112:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [19836928, 19841024) 0 0 00000080:00200000:1.0:1789760743.819117:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760743.819130:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4843, 4843] 00020000:00200000:1.0:1789760743.819137:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 19836928 chunk: [19836928, 19841024) 19841024, 4096 00020000:00200000:1.0:1789760743.819140:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760743.819155:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4843, 4843] 00000008:00010000:1.0:1789760743.819169:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760743.819173:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760743.819182:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [19836928, 19841024) append 0 append lockpos 0 00000080:00200000:1.0:1789760743.819193:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=19836928=0x12eb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760743.819239:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760743.819242:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 19841024 00000020:00010000:1.0:1789760743.819250:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760743.819253:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760743.819256:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5c8b3f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760743.819259:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760743.819262:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760743.819268:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760743.819274:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760743.819289:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760743.819368:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 19836928<>19841024 00000100:00100000:1.0:1789760743.878096:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972163 gen 3 00000008:00010000:1.0:1789760743.878143:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760743.878150:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.878158:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.878180:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760743.878203:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507251328:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.881919:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.938215:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972164 gen 3 00000008:00010000:1.0:1789760743.938256:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760743.938261:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.938270:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.938319:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760743.938347:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.943495:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507251584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.995137:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972165 gen 3 00000008:00010000:1.0:1789760743.995173:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760743.995179:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760743.997162:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760743.997207:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760743.997241:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507251584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760743.998275:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251712:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.036420:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972166 gen 3 00000008:00010000:1.0:1789760744.036456:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760744.036463:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.036472:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.036482:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760744.036507:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251712:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.037780:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760744.037799:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251840:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760744.087882:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760744.087893:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.087903:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.087960:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760744.087998:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.088902:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251968:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.135528:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972168 gen 3 00000008:00010000:1.0:1789760744.135575:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760744.135581:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.135589:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.135606:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760744.135638:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507251968:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.136425:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252096:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.181693:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972169 gen 3 00000008:00010000:1.0:1789760744.181730:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760744.181736:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.181746:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.181770:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760744.181794:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252096:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.182718:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507252224:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.238599:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972170 gen 3 00000008:00010000:1.0:1789760744.238634:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760744.238640:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.238651:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.238668:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760744.238700:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507252224:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.239692:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252352:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.286401:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972171 gen 3 00000008:00010000:1.0:1789760744.286441:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760744.286447:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.286458:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.286515:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760744.286550:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252352:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.290331:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507252480:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760744.323787:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760744.323972:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.323983:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.324003:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760744.324029:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507252480:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.325892:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252608:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.382221:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972172 gen 3 00000100:00100000:1.0:1789760744.382254:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b9180 x1876699507252608/t4294972173(4294972173) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760760 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760744.382293:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760744.382300:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.382310:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.382329:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760744.382349:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252608:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.382992:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760744.383005:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252736:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.425781:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972174 gen 3 00000008:00010000:1.0:1789760744.425824:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760744.425831:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.425840:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.425857:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760744.425877:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252736:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.426513:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252864:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760744.473607:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760744.473618:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.473627:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.473686:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760744.473710:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252864:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.524205:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972175 gen 3 00000100:00100000:1.0:1789760744.524236:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25bbb80 x1876699507252992/t4294972176(4294972176) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760760 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760744.524271:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760744.524277:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.524285:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.524303:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760744.524322:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507252992:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.525023:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253120:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.568843:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972177 gen 3 00000008:00010000:1.0:1789760744.568891:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760744.568908:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.568918:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.568934:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760744.568956:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253120:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.569745:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760744.569759:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.606096:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972178 gen 3 00000008:00010000:1.0:1789760744.606138:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760744.606144:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.606154:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.606175:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760744.606203:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.606787:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253376:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.644832:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972179 gen 3 00000008:00010000:1.0:1789760744.644864:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760744.644870:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.644878:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.644895:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760744.644917:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253376:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.645558:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760744.645572:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253504:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760744.683926:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760744.683938:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.683948:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.683978:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760744.684009:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253504:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.684756:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253632:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.728039:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972181 gen 3 00000008:00010000:1.0:1789760744.728082:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760744.728088:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.728098:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.728114:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760744.728135:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253632:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.728826:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760744.728841:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253760:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760744.778263:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760744.778275:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.778284:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.778304:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760744.778327:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253760:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.779218:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253888:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.816849:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972183 gen 3 00000008:00010000:1.0:1789760744.816892:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760744.816898:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.816906:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.816924:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760744.816947:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507253888:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.817720:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254016:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.890925:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972184 gen 3 00000008:00010000:1.0:1789760744.890982:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760744.890989:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.890998:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.891014:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760744.891037:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254016:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.891758:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254144:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.947609:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972185 gen 3 00000008:00010000:1.0:1789760744.947641:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760744.947648:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760744.947657:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760744.947674:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760744.947696:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254144:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760744.948371:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254272:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.003616:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972186 gen 3 00000008:00010000:1.0:1789760745.003650:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760745.003656:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.003667:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.003685:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760745.003707:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254272:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.005235:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254400:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.052183:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972187 gen 3 00000008:00010000:1.0:1789760745.052220:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760745.052227:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.052236:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.052255:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760745.052276:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254400:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.052890:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254528:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.099770:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972188 gen 3 00000008:00010000:1.0:1789760745.099804:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760745.099811:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.099820:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.099839:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760745.099862:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254528:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.100497:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.144446:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972189 gen 3 00000008:00010000:1.0:1789760745.144483:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760745.144489:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.144498:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.144514:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760745.144538:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.145493:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.183874:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972190 gen 3 00000008:00010000:1.0:1789760745.183943:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760745.183950:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.183968:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.183985:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760745.184068:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507254784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.184600:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254912:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.218423:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972191 gen 3 00000008:00010000:1.0:1789760745.218458:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760745.218465:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.218475:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.218537:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760745.218562:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507254912:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.219762:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255040:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760745.260576:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760745.260589:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.260600:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.260660:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760745.260684:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255040:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.264195:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255168:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.303390:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972193 gen 3 00000008:00010000:1.0:1789760745.303711:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760745.303719:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.303729:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.303751:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760745.303782:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255168:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.305157:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255296:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.341763:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972194 gen 3 00000008:00010000:1.0:1789760745.341802:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760745.341810:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.341820:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.341837:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760745.341860:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255296:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.343481:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255424:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.386482:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972195 gen 3 00000008:00010000:1.0:1789760745.386513:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760745.386519:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.386528:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.386544:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760745.386564:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255424:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.387111:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255552:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.423440:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972196 gen 3 00000008:00010000:1.0:1789760745.423475:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760745.423483:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.423492:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.423510:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760745.423532:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255552:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.424201:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255680:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.458819:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972197 gen 3 00000008:00010000:1.0:1789760745.458863:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760745.458869:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.458879:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.458903:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760745.458926:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255680:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.459661:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255808:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.503766:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972198 gen 3 00000008:00010000:1.0:1789760745.503799:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760745.503805:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.503815:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.503831:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760745.503852:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507255808:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.504791:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255936:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.537294:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972199 gen 3 00000008:00010000:1.0:1789760745.537332:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760745.537339:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.537349:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.537451:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760745.537475:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507255936:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.538725:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507256064:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760745.592157:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760745.592169:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.592179:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.592201:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760745.592225:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507256064:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.593108:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507256192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.651138:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972200 gen 3 00000100:00100000:1.0:1789760745.651167:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b9f80 x1876699507256192/t4294972201(4294972201) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760761 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760745.651203:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760745.651208:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.651217:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.651238:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760745.651257:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507256192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.714069:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972202 gen 3 00000008:00010000:1.0:1789760745.714116:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760745.714122:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.714131:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.714160:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760745.714183:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507256320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.715085:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507256448:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.816241:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972203 gen 3 00000008:00010000:1.0:1789760745.816277:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760745.816283:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.816292:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.816314:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760745.816338:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507256448:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.817077:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507256576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.884834:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972204 gen 3 00000008:00010000:1.0:1789760745.884871:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760745.884877:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.884886:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.884949:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760745.884982:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507256576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.891037:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760745.891056:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507256704:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760745.948702:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972205 gen 3 00000008:00010000:1.0:1789760745.948737:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760745.948743:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760745.948754:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760745.948811:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760745.948834:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507256704:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760745.951539:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20013056 i_blocks 40967 00000020:00200000:1.0:1789760745.951545:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760745.951557:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760745.951584:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20013056, count: 4096 00000080:00200000:1.0:1789760745.951591:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20013056, bytes: 4096 00000080:00200000:1.0:1789760745.951601:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760745.951609:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760745.951611:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20013056, 20017152) 0 0 00000080:00200000:1.0:1789760745.951615:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760745.951628:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4886, 4886] 00020000:00200000:1.0:1789760745.951635:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20013056 chunk: [20013056, 20017152) 20017152, 4096 00020000:00200000:1.0:1789760745.951639:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760745.951655:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4886, 4886] 00000008:00010000:1.0:1789760745.951668:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760745.951673:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760745.951681:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20013056, 20017152) append 0 append lockpos 0 00000080:00200000:1.0:1789760745.951692:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20013056=0x1316000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760745.951737:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760745.951741:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20017152 00000020:00010000:1.0:1789760745.951747:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760745.951751:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760745.951754:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5c8bf38/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760745.951757:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760745.951760:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760745.951767:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760745.951772:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760745.951786:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760745.951860:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 20013056<>20017152 00000100:00100000:1.0:1789760745.951900:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760745.951918:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507256832:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.012947:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972206 gen 3 00000008:00010000:1.0:1789760746.012991:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760746.012997:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.013006:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.013039:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760746.013064:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507256832:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760746.122063:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4888, 4888] 00000080:00200000:1.0:1789760746.122077:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760746.122081:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760746.122094:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20025344 i_blocks 40967 00000020:00200000:1.0:1789760746.122097:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760746.122105:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760746.122131:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20025344, count: 4096 00000080:00200000:1.0:1789760746.122137:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20025344, bytes: 4096 00000080:00200000:1.0:1789760746.122145:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760746.122153:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760746.122155:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20025344, 20029440) 0 0 00000080:00200000:1.0:1789760746.122158:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760746.122170:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4889, 4889] 00020000:00200000:1.0:1789760746.122177:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20025344 chunk: [20025344, 20029440) 20029440, 4096 00020000:00200000:1.0:1789760746.122180:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760746.122197:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4889, 4889] 00000008:00010000:1.0:1789760746.122211:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760746.122216:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760746.122224:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20025344, 20029440) append 0 append lockpos 0 00000080:00200000:1.0:1789760746.122235:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20025344=0x1319000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760746.122281:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760746.122283:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20029440 00000020:00010000:1.0:1789760746.122290:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760746.122294:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760746.122297:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5c8bf38/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760746.122300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be98a8 (ffff8d25c80385a8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760746.122302:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80385a8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760746.122308:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760746.122314:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760746.122328:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760746.122580:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 20025344<>20029440 00000100:00100000:1.0:1789760746.189601:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972209 gen 3 00000008:00010000:1.0:1789760746.189637:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760746.189643:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.189652:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.189668:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760746.189690:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507257216:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.254984:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972210 gen 3 00000008:00010000:1.0:1789760746.255028:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760746.255034:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.255044:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.255065:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760746.255090:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507257344:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.255678:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760746.255693:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507257472:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.295185:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972211 gen 3 00000008:00010000:1.0:1789760746.295223:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760746.295229:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.295238:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.295256:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760746.295280:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507257472:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.295869:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760746.295885:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507257600:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.333481:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972212 gen 3 00000008:00010000:1.0:1789760746.333516:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760746.333522:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.333532:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.333549:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760746.333574:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507257600:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.334373:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760746.334430:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507257728:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.372644:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972213 gen 3 00000008:00010000:1.0:1789760746.372679:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760746.372685:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.372694:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.372713:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760746.372737:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507257728:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.373344:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760746.373358:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507257856:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760746.413710:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760746.413721:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.413732:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.413752:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760746.413778:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507257856:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.414651:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507257984:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.452279:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972214 gen 3 00000100:00100000:1.0:1789760746.452309:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25ba300 x1876699507257984/t4294972215(4294972215) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760762 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760746.452345:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760746.452353:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.452362:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.452414:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760746.452435:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507257984:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.453002:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507258112:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.491852:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972216 gen 3 00000008:00010000:1.0:1789760746.491898:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760746.491904:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.491913:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.491932:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760746.491961:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507258112:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.492669:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760746.492684:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507258240:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.530662:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972217 gen 3 00000008:00010000:1.0:1789760746.530696:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760746.530702:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.530712:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.530730:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760746.530753:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507258240:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.531529:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760746.531546:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507258368:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.602282:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972218 gen 3 00000008:00010000:1.0:1789760746.602324:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760746.602331:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.602341:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.602351:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760746.602390:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507258368:192.168.203.141@tcp:4:dd.0 00000080:00200008:1.0:1789760746.602569:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20066304, count: 4096 00000080:00200000:1.0:1789760746.602577:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20066304, bytes: 4096 00000080:00200000:1.0:1789760746.602585:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760746.602592:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760746.602594:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20066304, 20070400) 0 0 00000080:00200000:1.0:1789760746.602598:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760746.602610:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4899, 4899] 00020000:00200000:1.0:1789760746.602617:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20066304 chunk: [20066304, 20070400) 20070400, 4096 00020000:00200000:1.0:1789760746.602621:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760746.602636:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4899, 4899] 00000008:00010000:1.0:1789760746.602650:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760746.602654:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760746.602663:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20066304, 20070400) append 0 append lockpos 0 00000080:00200000:1.0:1789760746.602674:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20066304=0x1323000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760746.602728:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760746.602731:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20070400 00000020:00010000:1.0:1789760746.602737:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760746.602740:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760746.602743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760746.602746:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760746.602749:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760746.602755:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760746.602762:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760746.602796:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760746.602884:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 20066304<>20070400 00000100:00100000:1.0:1789760746.602927:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760746.602941:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507258496:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760746.667702:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972219 gen 3 00000008:00010000:1.0:1789760746.667739:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760746.667745:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.667754:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.667782:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760746.667806:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507258496:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760746.722231:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4900, 4900] 00000080:00200000:1.0:1789760746.722253:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760746.722260:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760746.722274:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20074496 i_blocks 40967 00000020:00200000:1.0:1789760746.722279:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760746.722292:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760746.722319:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20074496, count: 4096 00000080:00200000:1.0:1789760746.722327:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20074496, bytes: 4096 00000080:00200000:1.0:1789760746.722338:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760746.722345:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760746.722347:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20074496, 20078592) 0 0 00000080:00200000:1.0:1789760746.722351:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760746.722364:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4901, 4901] 00020000:00200000:1.0:1789760746.722370:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20074496 chunk: [20074496, 20078592) 20078592, 4096 00020000:00200000:1.0:1789760746.722374:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760746.722408:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4901, 4901] 00000008:00010000:1.0:1789760746.722422:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760746.722426:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760746.722436:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20074496, 20078592) append 0 append lockpos 0 00000080:00200000:1.0:1789760746.722448:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20074496=0x1325000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760746.722503:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760746.722506:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20078592 00000020:00010000:1.0:1789760746.722513:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760746.722516:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760746.722519:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760746.722523:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760746.722526:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760746.722533:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760746.722538:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760746.722555:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760746.722633:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 20074496<>20078592 00000008:00010000:1.0:1789760746.790770:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760746.790781:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760746.790791:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760746.790875:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760746.790903:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507258752:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760747.138029:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4908, 4908] 00000080:00200000:1.0:1789760747.138042:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760747.138047:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760747.138059:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20107264 i_blocks 40967 00000020:00200000:1.0:1789760747.138062:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760747.138071:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760747.138099:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20107264, count: 4096 00000080:00200000:1.0:1789760747.138105:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20107264, bytes: 4096 00000080:00200000:1.0:1789760747.138113:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760747.138120:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760747.138122:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20107264, 20111360) 0 0 00000080:00200000:1.0:1789760747.138125:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760747.138137:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4909, 4909] 00020000:00200000:1.0:1789760747.138144:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20107264 chunk: [20107264, 20111360) 20111360, 4096 00020000:00200000:1.0:1789760747.138148:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760747.138163:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4909, 4909] 00000008:00010000:1.0:1789760747.138191:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760747.138196:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760747.138204:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20107264, 20111360) append 0 append lockpos 0 00000080:00200000:1.0:1789760747.138214:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20107264=0x132d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760747.138258:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760747.138260:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20111360 00000020:00010000:1.0:1789760747.138267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760747.138270:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.138273:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.138276:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760747.138278:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760747.138284:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760747.138289:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760747.138302:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760747.138372:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20107264<>20111360 00000100:00100000:1.0:1789760747.138444:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760747.138458:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507259776:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760747.203432:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972229 gen 3 00000008:00010000:1.0:1789760747.203484:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451540 00000020:00200000:1.0:1789760747.203490:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760747.203499:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760747.203528:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451540 00000100:00100000:1.0:1789760747.203551:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507259776:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760747.257326:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4910, 4910] 00000080:00200000:1.0:1789760747.257339:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760747.257345:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760747.257358:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20115456 i_blocks 40967 00000020:00200000:1.0:1789760747.257361:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760747.257370:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760747.257417:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20115456, count: 4096 00000080:00200000:1.0:1789760747.257424:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20115456, bytes: 4096 00000080:00200000:1.0:1789760747.257431:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760747.257506:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760747.257508:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20115456, 20119552) 0 0 00000080:00200000:1.0:1789760747.257512:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760747.257580:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4911, 4911] 00020000:00200000:1.0:1789760747.257590:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20115456 chunk: [20115456, 20119552) 20119552, 4096 00020000:00200000:1.0:1789760747.257593:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760747.257611:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4911, 4911] 00000008:00010000:1.0:1789760747.257626:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760747.257630:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760747.257639:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20115456, 20119552) append 0 append lockpos 0 00000080:00200000:1.0:1789760747.257651:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20115456=0x132f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760747.257699:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760747.257701:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20119552 00000020:00010000:1.0:1789760747.257708:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760747.257711:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.257714:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.257718:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760747.257720:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760747.257728:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760747.257736:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760747.257752:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451540 00000008:00100000:1.0:1789760747.257835:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20115456<>20119552 00000100:00100000:1.0:1789760747.265158:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507260416:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760747.265177:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507260288:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760747.265188:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e0668622-7dd7-45f4-a020-184f7e88cc06:2416:1876699507260544:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760747.274248:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2416:1876699507260160:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760747.274275:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e0668622-7dd7-45f4-a020-184f7e88cc06:2416:1876699507260672:192.168.203.141@tcp:400:kworker.0 00000080:00200000:1.0:1789760747.326537:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4911, 4911] 00000080:00200000:1.0:1789760747.326559:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760747.326565:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760747.326580:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20119552 i_blocks 40967 00000020:00200000:1.0:1789760747.326584:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760747.326596:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760747.326623:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20119552, count: 4096 00000080:00200000:1.0:1789760747.326630:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20119552, bytes: 4096 00000080:00200000:1.0:1789760747.326641:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760747.326648:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760747.326650:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20119552, 20123648) 0 0 00000080:00200000:1.0:1789760747.326653:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760747.326665:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4912, 4912] 00020000:00200000:1.0:1789760747.326672:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20119552 chunk: [20119552, 20123648) 20123648, 4096 00020000:00200000:1.0:1789760747.326675:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760747.326692:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4912, 4912] 00000008:00010000:1.0:1789760747.326706:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760747.326711:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760747.326931:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20119552, 20123648) append 0 append lockpos 0 00000080:00200000:1.0:1789760747.327015:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20119552=0x1330000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760747.327069:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760747.327071:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20123648 00000020:00010000:1.0:1789760747.327079:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760747.327082:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.327085:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.327089:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760747.327092:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760747.327099:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760747.327104:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760747.327128:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760747.327208:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 20119552<>20123648 00000100:00100000:1.0:1789760747.393440:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972231 gen 3 00000100:00100000:1.0:1789760747.393471:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8168e00 x1876699507260800/t4294972232(4294972232) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760763 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760747.393506:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760747.393512:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760747.393521:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760747.393590:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760747.393612:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507260800:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760747.526783:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4915, 4915] 00000080:00200000:1.0:1789760747.526796:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760747.526802:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760747.526814:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20135936 i_blocks 40967 00000020:00200000:1.0:1789760747.526818:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760747.526826:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760747.526852:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20135936, count: 4096 00000080:00200000:1.0:1789760747.526859:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20135936, bytes: 4096 00000080:00200000:1.0:1789760747.526867:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760747.526873:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760747.526875:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20135936, 20140032) 0 0 00000080:00200000:1.0:1789760747.526879:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760747.526892:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4916, 4916] 00020000:00200000:1.0:1789760747.526898:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20135936 chunk: [20135936, 20140032) 20140032, 4096 00020000:00200000:1.0:1789760747.526902:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760747.526916:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4916, 4916] 00000008:00010000:1.0:1789760747.526930:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760747.526935:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760747.526954:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20135936, 20140032) append 0 append lockpos 0 00000080:00200000:1.0:1789760747.526964:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20135936=0x1334000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760747.527008:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760747.527012:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20140032 00000020:00010000:1.0:1789760747.527019:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760747.527022:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.527025:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.527028:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760747.527031:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760747.527037:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760747.527259:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760747.527276:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760747.527351:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20135936<>20140032 00000100:00100000:1.0:1789760747.571431:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972236 gen 3 00000008:00010000:1.0:1789760747.571482:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760747.571488:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760747.571495:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760747.571516:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760747.571539:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507261312:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760747.683950:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4918, 4918] 00000080:00200000:1.0:1789760747.683967:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760747.683972:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760747.683988:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20148224 i_blocks 40967 00000020:00200000:1.0:1789760747.683992:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760747.684001:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760747.684037:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20148224, count: 4096 00000080:00200000:1.0:1789760747.684044:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20148224, bytes: 4096 00000080:00200000:1.0:1789760747.684055:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760747.684064:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760747.684066:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20148224, 20152320) 0 0 00000080:00200000:1.0:1789760747.684071:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760747.684089:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4919, 4919] 00020000:00200000:1.0:1789760747.684097:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20148224 chunk: [20148224, 20152320) 20152320, 4096 00020000:00200000:1.0:1789760747.684103:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760747.684122:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4919, 4919] 00000008:00010000:1.0:1789760747.684139:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760747.684145:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760747.684158:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20148224, 20152320) append 0 append lockpos 0 00000080:00200000:1.0:1789760747.684172:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20148224=0x1337000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760747.684231:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760747.684234:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20152320 00000020:00010000:1.0:1789760747.684242:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760747.684245:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.684249:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.684253:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760747.684255:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760747.684263:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760747.684269:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760747.684285:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760747.685008:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20148224<>20152320 00000100:00100000:1.0:1789760747.748821:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972239 gen 3 00000008:00010000:1.0:1789760747.748870:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760747.748877:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760747.748884:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760747.749400:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760747.749424:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507261696:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760747.750582:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20152320 i_blocks 40967 00000020:00200000:1.0:1789760747.750587:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760747.750597:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760747.750634:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20152320, count: 4096 00000080:00200000:1.0:1789760747.750642:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20152320, bytes: 4096 00000080:00200000:1.0:1789760747.750652:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760747.750659:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760747.750661:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20152320, 20156416) 0 0 00000080:00200000:1.0:1789760747.750665:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760747.750677:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4920, 4920] 00020000:00200000:1.0:1789760747.750685:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20152320 chunk: [20152320, 20156416) 20156416, 4096 00020000:00200000:1.0:1789760747.750689:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760747.750704:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4920, 4920] 00000008:00010000:1.0:1789760747.750718:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760747.750723:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760747.750731:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20152320, 20156416) append 0 append lockpos 0 00000080:00200000:1.0:1789760747.750742:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20152320=0x1338000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760747.750788:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760747.750792:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20156416 00000020:00010000:1.0:1789760747.750798:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760747.750801:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.750804:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760747.750807:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760747.750810:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760747.750816:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760747.750822:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760747.750860:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760747.750948:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20152320<>20156416 00000100:00100000:1.0:1789760747.750988:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760747.751003:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507261824:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760747.799979:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972240 gen 3 00000008:00010000:1.0:1789760747.800016:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760747.800022:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760747.800031:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760747.800093:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760747.800117:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507261824:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760748.120214:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4927, 4927] 00000080:00200000:1.0:1789760748.120229:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760748.120235:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760748.120251:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20185088 i_blocks 40967 00000020:00200000:1.0:1789760748.120255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760748.120266:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760748.120299:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20185088, count: 4096 00000080:00200000:1.0:1789760748.120307:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20185088, bytes: 4096 00000080:00200000:1.0:1789760748.120316:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760748.120324:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760748.120327:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20185088, 20189184) 0 0 00000080:00200000:1.0:1789760748.120331:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760748.120345:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4928, 4928] 00020000:00200000:1.0:1789760748.120354:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20185088 chunk: [20185088, 20189184) 20189184, 4096 00020000:00200000:1.0:1789760748.120358:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760748.120416:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4928, 4928] 00000008:00010000:1.0:1789760748.120435:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760748.120441:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760748.120453:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20185088, 20189184) append 0 append lockpos 0 00000080:00200000:1.0:1789760748.120465:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20185088=0x1340000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760748.120522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760748.120526:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20189184 00000020:00010000:1.0:1789760748.120534:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760748.120538:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760748.120541:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760748.120545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760748.120548:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760748.120556:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760748.120562:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760748.120578:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760748.120681:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20185088<>20189184 00000100:00100000:1.0:1789760748.174878:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972248 gen 3 00000008:00010000:1.0:1789760748.174932:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760748.174948:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.174960:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.175081:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760748.175114:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507262848:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760748.333356:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4931, 4931] 00000080:00200000:1.0:1789760748.334111:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760748.334122:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760748.334142:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20201472 i_blocks 40967 00000020:00200000:1.0:1789760748.334148:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760748.334166:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760748.334201:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20201472, count: 4096 00000080:00200000:1.0:1789760748.334209:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20201472, bytes: 4096 00000080:00200000:1.0:1789760748.334225:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760748.334236:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760748.334239:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20201472, 20205568) 0 0 00000080:00200000:1.0:1789760748.334244:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760748.334262:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4932, 4932] 00020000:00200000:1.0:1789760748.334271:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20201472 chunk: [20201472, 20205568) 20205568, 4096 00020000:00200000:1.0:1789760748.334275:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760748.334298:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4932, 4932] 00000008:00010000:1.0:1789760748.334318:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760748.334324:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760748.334335:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20201472, 20205568) append 0 append lockpos 0 00000080:00200000:1.0:1789760748.334349:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20201472=0x1344000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760748.334652:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760748.334655:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20205568 00000020:00010000:1.0:1789760748.334665:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760748.334670:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760748.334673:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760748.334677:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760748.334680:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760748.334690:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760748.334696:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760748.334724:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760748.334832:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20201472<>20205568 00000100:00100000:1.0:1789760748.420953:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972252 gen 3 00000008:00010000:1.0:1789760748.421020:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760748.421028:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.421049:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.421070:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760748.421101:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507263360:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760748.466372:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4933, 4933] 00000080:00200000:1.0:1789760748.466423:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760748.466431:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760748.466449:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20209664 i_blocks 40967 00000020:00200000:1.0:1789760748.466453:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760748.466463:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760748.466497:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20209664, count: 4096 00000080:00200000:1.0:1789760748.466504:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20209664, bytes: 4096 00000080:00200000:1.0:1789760748.466513:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760748.466522:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760748.466524:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20209664, 20213760) 0 0 00000080:00200000:1.0:1789760748.466528:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760748.466544:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4934, 4934] 00020000:00200000:1.0:1789760748.466553:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20209664 chunk: [20209664, 20213760) 20213760, 4096 00020000:00200000:1.0:1789760748.466557:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760748.466577:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4934, 4934] 00000008:00010000:1.0:1789760748.466595:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760748.466601:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760748.466613:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20209664, 20213760) append 0 append lockpos 0 00000080:00200000:1.0:1789760748.466627:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20209664=0x1346000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760748.466872:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760748.466875:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20213760 00000020:00010000:1.0:1789760748.466884:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760748.466888:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760748.466891:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760748.466948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760748.466952:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760748.466961:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760748.467126:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760748.467147:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760748.467248:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20209664<>20213760 00000008:00010000:1.0:1789760748.498665:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760748.498681:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.498693:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.498715:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760748.498748:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507263616:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.673525:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972258 gen 3 00000008:00010000:1.0:1789760748.673592:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760748.673600:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.673611:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.673631:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760748.673662:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507264128:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.675202:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264256:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.707477:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972259 gen 3 00000008:00010000:1.0:1789760748.707532:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760748.707540:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.707554:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.707568:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760748.707600:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264256:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760748.708282:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000008:00100000:1.0:1789760748.708460:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20234240<>20238336 00000100:00100000:1.0:1789760748.708518:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507264384:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.762180:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972260 gen 3 00000008:00010000:1.0:1789760748.762231:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760748.762240:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.762252:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.762276:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760748.762747:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507264384:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.764601:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264512:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760748.800038:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760748.800054:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.800068:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.800131:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760748.800164:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264512:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.801183:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507264640:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.849371:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972262 gen 3 00000008:00010000:1.0:1789760748.849458:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760748.849467:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.849479:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.849542:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760748.849574:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507264640:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.857556:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264768:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.918710:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972263 gen 3 00000008:00010000:1.0:1789760748.918759:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760748.918767:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.918779:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.918802:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760748.918832:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264768:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.919753:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760748.919774:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264896:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760748.957286:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760748.957303:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.957315:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760748.957436:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760748.957468:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507264896:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.958609:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760748.958632:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507265024:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760748.999755:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972265 gen 3 00000008:00010000:1.0:1789760748.999819:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760748.999827:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760748.999839:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760749.000017:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760749.000049:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507265024:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760749.235780:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972270 gen 3 00000008:00010000:1.0:1789760749.235832:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760749.235841:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760749.235854:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760749.235878:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760749.235912:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507265664:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760749.236535:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:1.0:1789760749.236639:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20279296<>20283392 00000100:00100000:1.0:1789760749.236691:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507265792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760749.298204:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972271 gen 3 00000008:00010000:1.0:1789760749.298256:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:1.0:1789760749.298265:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760749.298278:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760749.298302:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:1.0:1789760749.298333:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507265792:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760749.340509:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4952, 4952] 00000080:00200000:1.0:1789760749.340528:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760749.340535:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760749.340551:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20287488 i_blocks 40967 00000020:00200000:1.0:1789760749.340555:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760749.340565:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760749.340599:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20287488, count: 4096 00000080:00200000:1.0:1789760749.340606:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20287488, bytes: 4096 00000080:00200000:1.0:1789760749.340617:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760749.340626:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760749.340628:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20287488, 20291584) 0 0 00000080:00200000:1.0:1789760749.340643:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760749.340662:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4953, 4953] 00020000:00200000:1.0:1789760749.340672:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20287488 chunk: [20287488, 20291584) 20291584, 4096 00020000:00200000:1.0:1789760749.340676:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760749.340699:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4953, 4953] 00000008:00010000:1.0:1789760749.340718:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760749.340724:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760749.340735:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20287488, 20291584) append 0 append lockpos 0 00000080:00200000:1.0:1789760749.340750:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20287488=0x1359000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760749.340811:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760749.340814:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20291584 00000020:00010000:1.0:1789760749.340822:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760749.340826:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760749.340829:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760749.340832:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760749.340835:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760749.340843:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760749.340850:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760749.340868:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760749.341057:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20287488<>20291584 00000100:00100000:1.0:1789760749.341104:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760749.341126:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507266048:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760749.372498:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760749.372514:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760749.372525:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760749.372544:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760749.372567:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507266048:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.224028:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972287 gen 3 00000008:00010000:1.0:1789760750.224064:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760750.224071:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.224081:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.224167:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760750.224194:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507267840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.227054:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507267968:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760750.277494:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760750.277505:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.277515:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.277592:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760750.277616:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507267968:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760750.405806:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4970, 4970] 00000080:00200000:1.0:1789760750.405819:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760750.405825:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760750.405838:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20361216 i_blocks 40967 00000020:00200000:1.0:1789760750.405842:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760750.405850:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760750.405877:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20361216, count: 4096 00000080:00200000:1.0:1789760750.405883:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20361216, bytes: 4096 00000080:00200000:1.0:1789760750.405895:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760750.405902:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760750.405904:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20361216, 20365312) 0 0 00000080:00200000:1.0:1789760750.405908:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760750.405933:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4971, 4971] 00020000:00200000:1.0:1789760750.405940:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20361216 chunk: [20361216, 20365312) 20365312, 4096 00020000:00200000:1.0:1789760750.405944:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760750.405959:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4971, 4971] 00000008:00010000:1.0:1789760750.406094:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760750.406099:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760750.406110:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20361216, 20365312) append 0 append lockpos 0 00000080:00200000:1.0:1789760750.406122:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20361216=0x136b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760750.406172:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760750.406175:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20365312 00000020:00010000:1.0:1789760750.406182:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760750.406186:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760750.406189:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760750.406192:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760750.406195:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760750.406202:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760750.406208:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760750.406222:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760750.406297:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20361216<>20365312 00000100:00100000:1.0:1789760750.406413:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760750.406729:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268352:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.458882:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972291 gen 3 00000008:00010000:1.0:1789760750.459006:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760750.459013:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.459022:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.459081:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760750.459106:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268352:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.462236:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268480:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.524119:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972292 gen 3 00000008:00010000:1.0:1789760750.524154:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760750.524160:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.524169:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.524190:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760750.524213:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268480:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.524821:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760750.524833:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268608:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.585497:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972293 gen 3 00000008:00010000:1.0:1789760750.585536:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760750.585542:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.585551:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.585571:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760750.585594:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268608:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.586131:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268736:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.680155:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972294 gen 3 00000008:00010000:1.0:1789760750.680193:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760750.680200:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.680209:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.680231:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760750.680254:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268736:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.680811:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507268864:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.726850:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972295 gen 3 00000008:00010000:1.0:1789760750.726890:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760750.726909:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.726927:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.726945:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760750.726977:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507268864:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.727777:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268992:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760750.764487:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760750.764499:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.764507:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.764523:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760750.764546:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507268992:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.765406:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760750.765427:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507269120:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.813208:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972297 gen 3 00000008:00010000:1.0:1789760750.813253:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760750.813259:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.813266:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.813315:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760750.813334:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507269120:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.814120:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507269248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.850846:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972298 gen 3 00000008:00010000:1.0:1789760750.850881:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760750.850886:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.850893:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.850906:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760750.850941:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507269248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.851847:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507269376:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760750.967175:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972299 gen 3 00000008:00010000:1.0:1789760750.967212:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760750.967219:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760750.967229:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760750.967315:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760750.967340:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507269376:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760751.141656:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4981, 4981] 00000080:00200000:1.0:1789760751.141674:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760751.141680:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760751.141719:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20406272 i_blocks 40967 00000020:00200000:1.0:1789760751.141725:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760751.141736:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760751.141762:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20406272, count: 4096 00000080:00200000:1.0:1789760751.141768:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20406272, bytes: 4096 00000080:00200000:1.0:1789760751.141778:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760751.141785:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760751.141787:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20406272, 20410368) 0 0 00000080:00200000:1.0:1789760751.141791:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760751.141803:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4982, 4982] 00020000:00200000:1.0:1789760751.141809:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20406272 chunk: [20406272, 20410368) 20410368, 4096 00020000:00200000:1.0:1789760751.141812:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760751.141828:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4982, 4982] 00000008:00010000:1.0:1789760751.141843:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760751.141848:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760751.141857:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20406272, 20410368) append 0 append lockpos 0 00000080:00200000:1.0:1789760751.141869:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20406272=0x1376000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760751.141923:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760751.141925:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20410368 00000020:00010000:1.0:1789760751.141932:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760751.141935:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760751.141938:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760751.141941:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760751.141943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760751.141950:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760751.141955:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760751.141968:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760751.142046:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20406272<>20410368 00000008:00010000:1.0:1789760751.241962:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760751.241974:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760751.241984:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760751.242010:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760751.242034:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507269760:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760751.635760:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4990, 4990] 00000080:00200000:1.0:1789760751.635773:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760751.635779:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760751.635790:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20443136 i_blocks 40967 00000020:00200000:1.0:1789760751.635794:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760751.635801:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760751.635827:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20443136, count: 4096 00000080:00200000:1.0:1789760751.635833:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20443136, bytes: 4096 00000080:00200000:1.0:1789760751.635841:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760751.635848:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760751.635850:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20443136, 20447232) 0 0 00000080:00200000:1.0:1789760751.635854:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760751.635866:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4991, 4991] 00020000:00200000:1.0:1789760751.635872:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20443136 chunk: [20443136, 20447232) 20447232, 4096 00020000:00200000:1.0:1789760751.635876:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760751.635890:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4991, 4991] 00000008:00010000:1.0:1789760751.635924:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760751.635929:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760751.635938:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20443136, 20447232) append 0 append lockpos 0 00000080:00200000:1.0:1789760751.635958:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20443136=0x137f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760751.635999:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760751.636002:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20447232 00000020:00010000:1.0:1789760751.636008:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_release():245 00000020:00010000:1.0:1789760751.636012:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760751.636015:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760751.636018:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c7e30038 (ffff8d25c5c8b998/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760751.636020:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c5c8b998/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760751.636027:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760751.636032:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760751.636046:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760751.636118:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20443136<>20447232 00000100:00100000:1.0:1789760751.636155:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760751.636168:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507270912:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760751.695417:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972311 gen 3 00000008:00010000:1.0:1789760751.695464:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760751.695470:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760751.695477:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760751.695538:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760751.695561:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507270912:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760751.696180:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507271040:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760751.754861:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972312 gen 3 00000008:00010000:1.0:1789760751.754899:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760751.754905:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760751.754924:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760751.754941:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760751.754963:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507271040:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760751.755599:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7600 00000008:00100000:1.0:1789760751.755801:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20451328<>20455424 00000100:00100000:1.0:1789760751.755833:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760751.755845:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507271168:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760751.823343:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972313 gen 3 00000008:00010000:1.0:1789760751.823421:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760751.823427:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760751.823437:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760751.823458:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760751.823480:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507271168:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760751.880953:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760751.880964:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760751.880974:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760751.880996:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760751.881134:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507271296:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760752.035802:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [4997, 4997] 00000080:00200000:1.0:1789760752.035819:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760752.035824:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760752.035838:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20471808 i_blocks 40967 00000020:00200000:1.0:1789760752.035842:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760752.035851:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760752.035876:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20471808, count: 4096 00000080:00200000:1.0:1789760752.035881:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20471808, bytes: 4096 00000080:00200000:1.0:1789760752.035888:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760752.035915:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760752.035917:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20471808, 20475904) 0 0 00000080:00200000:1.0:1789760752.035921:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760752.035932:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [4998, 4998] 00020000:00200000:1.0:1789760752.035939:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20471808 chunk: [20471808, 20475904) 20475904, 4096 00020000:00200000:1.0:1789760752.036088:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760752.036105:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [4998, 4998] 00000008:00010000:1.0:1789760752.036118:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760752.036122:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760752.036131:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20471808, 20475904) append 0 append lockpos 0 00000080:00200000:1.0:1789760752.036143:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20471808=0x1386000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760752.036194:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760752.036197:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20475904 00000020:00010000:1.0:1789760752.036203:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760752.036206:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.036209:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.036213:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760752.036216:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760752.036222:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760752.036227:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760752.036246:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760752.036331:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20471808<>20475904 00000100:00100000:1.0:1789760752.036372:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760752.036422:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507271808:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760752.091823:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972318 gen 3 00000008:00010000:1.0:1789760752.091861:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760752.091867:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760752.091877:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760752.091902:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760752.091934:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507271808:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760752.206750:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972319 gen 3 00000008:00010000:1.0:1789760752.206792:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760752.206799:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760752.206809:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760752.206825:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760752.206849:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507271936:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760752.207939:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507272064:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760752.317697:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760752.317707:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760752.317716:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760752.317736:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760752.317757:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507272064:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760752.379731:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2417:1876699507272448:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.379814:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507272704:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.379836:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507272960:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.379859:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 2 async RPCs [0->1] 00000100:00100000:1.0:1789760752.379864:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507272576:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.379880:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507272832:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.385096:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507272576:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.385112:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507272832:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.385121:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2417:1876699507272448:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.385132:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507272704:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760752.385141:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507272960:192.168.203.141@tcp:400:kworker.0 00000080:00200000:1.0:1789760752.578533:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5007, 5007] 00000080:00200000:1.0:1789760752.578546:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760752.578551:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760752.578563:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20512768 i_blocks 40967 00000020:00200000:1.0:1789760752.578566:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760752.578574:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760752.578596:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20512768, count: 4096 00000080:00200000:1.0:1789760752.578603:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20512768, bytes: 4096 00000080:00200000:1.0:1789760752.578611:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760752.578618:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760752.578621:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20512768, 20516864) 0 0 00000080:00200000:1.0:1789760752.578625:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760752.578638:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5008, 5008] 00020000:00200000:1.0:1789760752.578644:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20512768 chunk: [20512768, 20516864) 20516864, 4096 00020000:00200000:1.0:1789760752.578648:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760752.578662:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5008, 5008] 00000008:00010000:1.0:1789760752.578675:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760752.578680:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760752.578689:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20512768, 20516864) append 0 append lockpos 0 00000080:00200000:1.0:1789760752.578702:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20512768=0x1390000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760752.578745:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760752.578748:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20516864 00000020:00010000:1.0:1789760752.578755:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760752.578758:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.578761:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.578764:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760752.578766:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760752.578773:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760752.578779:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760752.578792:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760752.578869:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20512768<>20516864 00000100:00100000:1.0:1789760752.578916:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760752.578928:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507273728:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760752.615708:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972328 gen 3 00000008:00010000:1.0:1789760752.615743:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760752.615749:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760752.615757:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760752.615779:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760752.615801:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507273728:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760752.774122:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5012, 5012] 00000080:00200000:1.0:1789760752.774141:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760752.774147:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760752.774162:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20533248 i_blocks 40967 00000020:00200000:1.0:1789760752.774166:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760752.774179:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760752.774210:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20533248, count: 4096 00000080:00200000:1.0:1789760752.774217:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20533248, bytes: 4096 00000080:00200000:1.0:1789760752.774228:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760752.774236:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760752.774238:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20533248, 20537344) 0 0 00000080:00200000:1.0:1789760752.774242:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760752.774255:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5013, 5013] 00020000:00200000:1.0:1789760752.774262:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20533248 chunk: [20533248, 20537344) 20537344, 4096 00020000:00200000:1.0:1789760752.774266:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760752.774282:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5013, 5013] 00000008:00010000:1.0:1789760752.774297:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760752.774302:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760752.774312:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20533248, 20537344) append 0 append lockpos 0 00000080:00200000:1.0:1789760752.774324:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20533248=0x1395000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760752.774532:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760752.774535:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20537344 00000020:00010000:1.0:1789760752.774543:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760752.774546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.774549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.774552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760752.774554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760752.774561:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760752.774567:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760752.774588:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760752.774671:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20533248<>20537344 00000100:00100000:1.0:1789760752.824000:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972333 gen 3 00000008:00010000:1.0:1789760752.824033:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760752.824040:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760752.824048:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760752.824071:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760752.824096:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507274368:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760752.867630:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5014, 5014] 00000080:00200000:1.0:1789760752.867644:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760752.867650:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760752.867661:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20541440 i_blocks 40967 00000020:00200000:1.0:1789760752.867665:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760752.867673:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760752.867710:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20541440, count: 4096 00000080:00200000:1.0:1789760752.867716:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20541440, bytes: 4096 00000080:00200000:1.0:1789760752.867724:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760752.867733:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760752.867735:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20541440, 20545536) 0 0 00000080:00200000:1.0:1789760752.867739:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760752.867751:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5015, 5015] 00020000:00200000:1.0:1789760752.867758:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20541440 chunk: [20541440, 20545536) 20545536, 4096 00020000:00200000:1.0:1789760752.867761:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760752.867777:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5015, 5015] 00000008:00010000:1.0:1789760752.867791:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760752.867797:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760752.867806:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20541440, 20545536) append 0 append lockpos 0 00000080:00200000:1.0:1789760752.867818:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20541440=0x1397000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760752.867864:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760752.867867:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20545536 00000020:00010000:1.0:1789760752.867874:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760752.867877:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.867880:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.867883:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760752.867885:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760752.867891:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760752.867897:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760752.867920:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760752.868004:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20541440<>20545536 00000100:00100000:1.0:1789760752.868042:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760752.868055:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507274624:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760752.905882:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972335 gen 3 00000008:00010000:1.0:1789760752.905930:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760752.905937:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760752.905944:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760752.905973:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760752.905997:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507274624:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760752.957935:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5016, 5016] 00000080:00200000:1.0:1789760752.957948:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760752.957953:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760752.957965:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20549632 i_blocks 40967 00000020:00200000:1.0:1789760752.957968:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760752.957976:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760752.958014:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20549632, count: 4096 00000080:00200000:1.0:1789760752.958020:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20549632, bytes: 4096 00000080:00200000:1.0:1789760752.958028:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760752.958035:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760752.958037:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20549632, 20553728) 0 0 00000080:00200000:1.0:1789760752.958041:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760752.958053:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5017, 5017] 00020000:00200000:1.0:1789760752.958060:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20549632 chunk: [20549632, 20553728) 20553728, 4096 00020000:00200000:1.0:1789760752.958064:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760752.958079:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5017, 5017] 00000008:00010000:1.0:1789760752.958091:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760752.958096:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760752.958104:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20549632, 20553728) append 0 append lockpos 0 00000080:00200000:1.0:1789760752.958114:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20549632=0x1399000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760752.958156:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760752.958159:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20553728 00000020:00010000:1.0:1789760752.958166:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760752.958169:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.958172:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760752.958175:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760752.958178:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760752.958184:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760752.958190:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760752.958202:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760752.958274:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20549632<>20553728 00000100:00100000:1.0:1789760752.996476:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972337 gen 3 00000008:00010000:1.0:1789760752.996514:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760752.996520:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760752.996529:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760752.996563:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760752.996752:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507274880:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760753.075863:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5019, 5019] 00000080:00200000:1.0:1789760753.075876:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760753.075880:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760753.075893:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20561920 i_blocks 40967 00000020:00200000:1.0:1789760753.075895:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760753.075914:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760753.075943:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20561920, count: 4096 00000080:00200000:1.0:1789760753.075950:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20561920, bytes: 4096 00000080:00200000:1.0:1789760753.075958:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760753.075966:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760753.075968:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20561920, 20566016) 0 0 00000080:00200000:1.0:1789760753.075972:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760753.075985:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5020, 5020] 00020000:00200000:1.0:1789760753.075993:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20561920 chunk: [20561920, 20566016) 20566016, 4096 00020000:00200000:1.0:1789760753.075997:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760753.076015:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5020, 5020] 00000008:00010000:1.0:1789760753.076030:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760753.076035:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760753.076046:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20561920, 20566016) append 0 append lockpos 0 00000080:00200000:1.0:1789760753.076059:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20561920=0x139c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760753.076107:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760753.076109:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20566016 00000020:00010000:1.0:1789760753.076115:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760753.076117:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760753.076119:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760753.076121:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760753.076123:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760753.076127:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760753.076131:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760753.076142:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760753.076206:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20561920<>20566016 00000100:00100000:1.0:1789760753.076236:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760753.076248:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275264:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.130065:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972340 gen 3 00000008:00010000:1.0:1789760753.130105:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760753.130112:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.130120:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.130173:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760753.130196:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275264:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.131034:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275392:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.221940:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972341 gen 3 00000008:00010000:1.0:1789760753.221987:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760753.221993:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.222002:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.222022:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760753.222044:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275392:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.292333:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972342 gen 3 00000008:00010000:1.0:1789760753.292413:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760753.292419:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.292428:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.292444:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760753.292465:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275520:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.293157:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.352577:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972343 gen 3 00000008:00010000:1.0:1789760753.352860:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760753.352866:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.352875:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.352945:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760753.352968:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.354018:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760753.354035:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275776:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760753.440953:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760753.440964:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.440973:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.441035:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760753.441060:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275776:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.441937:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.504605:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972345 gen 3 00000008:00010000:1.0:1789760753.504658:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760753.504665:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.504674:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.504694:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760753.504719:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507275904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.505640:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760753.505656:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276032:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.544250:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972346 gen 3 00000008:00010000:1.0:1789760753.544287:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760753.544294:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.544303:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.544321:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760753.544344:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276032:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.546661:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276160:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.627370:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972347 gen 3 00000008:00010000:1.0:1789760753.627421:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760753.627428:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.627438:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.627535:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760753.627558:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276160:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.629338:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507276288:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760753.712350:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760753.712362:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.712373:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.712416:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760753.712439:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507276288:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.713126:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276416:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.748273:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972349 gen 3 00000008:00010000:1.0:1789760753.748312:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760753.748319:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.748326:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.748348:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760753.748369:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276416:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.749147:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760753.749160:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276544:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.779599:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972350 gen 3 00000008:00010000:1.0:1789760753.779636:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760753.779642:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.779652:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.779671:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760753.779694:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276544:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.780324:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760753.780335:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507276672:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.826737:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972351 gen 3 00000008:00010000:1.0:1789760753.826772:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760753.826779:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.826788:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.826846:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760753.826874:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507276672:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.827359:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507276800:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760753.862304:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760753.862315:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.862324:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.862465:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760753.862490:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507276800:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.863245:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276928:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.904990:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972353 gen 3 00000008:00010000:1.0:1789760753.905120:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760753.905126:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.905136:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.905196:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760753.905220:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507276928:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.906202:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277056:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760753.951753:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760753.951764:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760753.951774:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760753.951794:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760753.951816:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277056:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760753.952655:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760753.952671:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277184:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760754.007663:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972355 gen 3 00000008:00010000:1.0:1789760754.007706:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760754.007713:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760754.007722:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760754.007740:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760754.007771:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277184:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760754.008280:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277312:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760754.065278:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972356 gen 3 00000008:00010000:1.0:1789760754.065318:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760754.065324:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760754.065334:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760754.065472:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760754.065495:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277312:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760754.071317:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760754.071340:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277440:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760754.125904:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760754.125914:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760754.125925:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760754.125982:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760754.126005:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277440:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760754.129253:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20635648 i_blocks 40967 00000020:00200000:1.0:1789760754.129259:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760754.129273:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760754.129300:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20635648, count: 4096 00000080:00200000:1.0:1789760754.129307:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20635648, bytes: 4096 00000080:00200000:1.0:1789760754.129317:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760754.129325:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760754.129327:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20635648, 20639744) 0 0 00000080:00200000:1.0:1789760754.129331:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760754.129421:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5038, 5038] 00020000:00200000:1.0:1789760754.129429:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20635648 chunk: [20635648, 20639744) 20639744, 4096 00020000:00200000:1.0:1789760754.129433:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760754.129450:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5038, 5038] 00000008:00010000:1.0:1789760754.129465:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760754.129470:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760754.129479:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20635648, 20639744) append 0 append lockpos 0 00000080:00200000:1.0:1789760754.129490:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20635648=0x13ae000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760754.129538:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760754.129541:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20639744 00000020:00010000:1.0:1789760754.129547:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760754.129551:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760754.129554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54176c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760754.129557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760754.129560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760754.129566:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760754.129572:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760754.129614:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760754.129692:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20635648<>20639744 00000100:00100000:1.0:1789760754.188174:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972358 gen 3 00000008:00010000:1.0:1789760754.188221:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760754.188228:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760754.188236:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760754.188252:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760754.188274:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277568:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760754.227791:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5039, 5039] 00000080:00200000:1.0:1789760754.227805:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760754.227811:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760754.227823:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20643840 i_blocks 40967 00000020:00200000:1.0:1789760754.227827:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760754.227836:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760754.227862:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20643840, count: 4096 00000080:00200000:1.0:1789760754.227868:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20643840, bytes: 4096 00000080:00200000:1.0:1789760754.227875:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760754.227883:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760754.227885:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20643840, 20647936) 0 0 00000080:00200000:1.0:1789760754.227901:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760754.227914:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5040, 5040] 00020000:00200000:1.0:1789760754.227921:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20643840 chunk: [20643840, 20647936) 20647936, 4096 00020000:00200000:1.0:1789760754.227924:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760754.227946:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5040, 5040] 00000008:00010000:1.0:1789760754.227959:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760754.227964:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760754.227973:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20643840, 20647936) append 0 append lockpos 0 00000080:00200000:1.0:1789760754.227983:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20643840=0x13b0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760754.228026:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760754.228029:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20647936 00000020:00010000:1.0:1789760754.228036:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760754.228039:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760754.228042:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54176c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760754.228046:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760754.228048:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760754.228055:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760754.228060:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760754.228076:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760754.228156:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20643840<>20647936 00000100:00100000:1.0:1789760754.228193:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760754.228206:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277824:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760754.271039:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972360 gen 3 00000008:00010000:1.0:1789760754.271076:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760754.271082:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760754.271091:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760754.271167:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760754.271192:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507277824:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760755.125823:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5057, 5057] 00000080:00200000:1.0:1789760755.125837:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760755.125841:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760755.191649:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5058, 5058] 00000080:00200000:1.0:1789760755.191662:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760755.191667:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760755.191679:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20721664 i_blocks 40967 00000020:00200000:1.0:1789760755.191683:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760755.191691:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760755.191718:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20721664, count: 4096 00000080:00200000:1.0:1789760755.191780:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20721664, bytes: 4096 00000080:00200000:1.0:1789760755.191790:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760755.191797:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760755.191800:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20721664, 20725760) 0 0 00000080:00200000:1.0:1789760755.191804:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760755.191817:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5059, 5059] 00020000:00200000:1.0:1789760755.191823:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20721664 chunk: [20721664, 20725760) 20725760, 4096 00020000:00200000:1.0:1789760755.191827:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760755.191841:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5059, 5059] 00000008:00010000:1.0:1789760755.191855:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760755.191860:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760755.191870:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20721664, 20725760) append 0 append lockpos 0 00000080:00200000:1.0:1789760755.191892:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20721664=0x13c3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760755.191939:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760755.191942:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20725760 00000020:00010000:1.0:1789760755.191950:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760755.191953:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.191956:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.191960:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760755.191962:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760755.191969:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760755.191975:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760755.191990:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760755.192254:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20721664<>20725760 00000100:00100000:1.0:1789760755.282688:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972379 gen 3 00000008:00010000:1.0:1789760755.282724:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760755.282730:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760755.282739:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760755.282898:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760755.282922:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507280256:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760755.332245:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5060, 5060] 00000080:00200000:1.0:1789760755.332257:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760755.332262:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760755.332273:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20729856 i_blocks 40967 00000020:00200000:1.0:1789760755.332277:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760755.332284:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760755.332310:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20729856, count: 4096 00000080:00200000:1.0:1789760755.332316:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20729856, bytes: 4096 00000080:00200000:1.0:1789760755.332323:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760755.332342:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760755.332344:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20729856, 20733952) 0 0 00000080:00200000:1.0:1789760755.332348:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760755.332359:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5061, 5061] 00020000:00200000:1.0:1789760755.332365:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20729856 chunk: [20729856, 20733952) 20733952, 4096 00020000:00200000:1.0:1789760755.332369:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760755.332409:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5061, 5061] 00000008:00010000:1.0:1789760755.332424:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760755.332429:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760755.332439:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20729856, 20733952) append 0 append lockpos 0 00000080:00200000:1.0:1789760755.332449:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20729856=0x13c5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760755.332491:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760755.332493:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20733952 00000020:00010000:1.0:1789760755.332500:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760755.332503:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.332505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.332508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760755.332511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760755.332517:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760755.332647:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760755.332663:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760755.332738:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 20729856<>20733952 00000008:00010000:1.0:1789760755.381666:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760755.381677:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760755.381686:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760755.381709:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760755.381734:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507280512:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760755.430288:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5062, 5062] 00000080:00200000:1.0:1789760755.430301:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760755.430305:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760755.430318:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20738048 i_blocks 40967 00000020:00200000:1.0:1789760755.430322:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760755.430330:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760755.430352:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20738048, count: 4096 00000080:00200000:1.0:1789760755.430358:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20738048, bytes: 4096 00000080:00200000:1.0:1789760755.430365:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760755.430374:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760755.430400:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20738048, 20742144) 0 0 00000080:00200000:1.0:1789760755.430404:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760755.430426:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5063, 5063] 00020000:00200000:1.0:1789760755.430433:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20738048 chunk: [20738048, 20742144) 20742144, 4096 00020000:00200000:1.0:1789760755.430444:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760755.430459:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5063, 5063] 00000008:00010000:1.0:1789760755.430473:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760755.430477:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760755.430485:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20738048, 20742144) append 0 append lockpos 0 00000080:00200000:1.0:1789760755.430496:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20738048=0x13c7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760755.430536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760755.430539:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20742144 00000020:00010000:1.0:1789760755.430545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760755.430548:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.430550:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.430553:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760755.430555:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760755.430561:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760755.430566:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760755.430579:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760755.430650:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20738048<>20742144 00000100:00100000:1.0:1789760755.430692:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760755.430707:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507280768:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760755.464321:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972383 gen 3 00000008:00010000:1.0:1789760755.464368:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760755.464374:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760755.464409:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760755.464424:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760755.464445:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507280768:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760755.495515:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5064, 5064] 00000080:00200000:1.0:1789760755.495527:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760755.495531:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760755.495544:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20746240 i_blocks 40967 00000020:00200000:1.0:1789760755.495547:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760755.495553:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760755.495574:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20746240, count: 4096 00000080:00200000:1.0:1789760755.495578:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20746240, bytes: 4096 00000080:00200000:1.0:1789760755.495584:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760755.495589:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760755.495591:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20746240, 20750336) 0 0 00000080:00200000:1.0:1789760755.495594:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760755.495603:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5065, 5065] 00020000:00200000:1.0:1789760755.495614:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20746240 chunk: [20746240, 20750336) 20750336, 4096 00020000:00200000:1.0:1789760755.495617:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760755.495631:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5065, 5065] 00000008:00010000:1.0:1789760755.495641:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760755.495645:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760755.495652:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20746240, 20750336) append 0 append lockpos 0 00000080:00200000:1.0:1789760755.495660:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20746240=0x13c9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760755.495701:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760755.495807:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20750336 00000020:00010000:1.0:1789760755.495816:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760755.495818:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.495821:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.495824:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760755.495827:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760755.495834:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760755.495840:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760755.495853:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760755.495940:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20746240<>20750336 00000100:00100000:1.0:1789760755.531794:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972385 gen 3 00000008:00010000:1.0:1789760755.531832:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760755.531838:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760755.531846:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760755.532017:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760755.532040:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507281024:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760755.888067:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5072, 5072] 00000080:00200000:1.0:1789760755.888081:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760755.888086:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760755.888097:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20779008 i_blocks 40967 00000020:00200000:1.0:1789760755.888112:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760755.888120:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760755.888146:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20779008, count: 4096 00000080:00200000:1.0:1789760755.888152:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20779008, bytes: 4096 00000080:00200000:1.0:1789760755.888160:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760755.888167:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760755.888169:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20779008, 20783104) 0 0 00000080:00200000:1.0:1789760755.888173:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760755.888185:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5073, 5073] 00020000:00200000:1.0:1789760755.888191:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20779008 chunk: [20779008, 20783104) 20783104, 4096 00020000:00200000:1.0:1789760755.888195:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760755.888210:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5073, 5073] 00000008:00010000:1.0:1789760755.888223:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760755.888227:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760755.888235:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20779008, 20783104) append 0 append lockpos 0 00000080:00200000:1.0:1789760755.888246:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20779008=0x13d1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760755.888291:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760755.888294:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20783104 00000020:00010000:1.0:1789760755.888300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760755.888303:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.888307:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760755.888310:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760755.888313:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760755.888319:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760755.888324:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760755.888337:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760755.888698:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20779008<>20783104 00000100:00100000:1.0:1789760755.951975:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972393 gen 3 00000008:00010000:1.0:1789760755.952086:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760755.952093:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760755.952101:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760755.952118:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760755.952140:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507282048:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760755.952751:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760755.952766:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507282176:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760755.986083:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972394 gen 3 00000008:00010000:1.0:1789760755.986122:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760755.986127:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760755.986136:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760755.986233:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760755.986257:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507282176:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760755.987358:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507282304:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.039446:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972395 gen 3 00000008:00010000:1.0:1789760756.039481:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760756.039487:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760756.039497:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760756.039520:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760756.039543:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507282304:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.113029:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972396 gen 3 00000008:00010000:1.0:1789760756.113339:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760756.113346:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760756.113361:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760756.113473:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760756.113497:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507282432:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.204315:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507282816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.244505:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972399 gen 3 00000008:00010000:1.0:1789760756.244541:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760756.244547:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760756.244555:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760756.244576:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760756.244599:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507282816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.469071:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507283456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.501678:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972404 gen 3 00000008:00010000:1.0:1789760756.501713:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760756.501720:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760756.501730:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760756.501796:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760756.501819:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507283456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.503819:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507283584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.559538:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972405 gen 3 00000008:00010000:1.0:1789760756.559590:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760756.559599:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760756.559613:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760756.559639:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760756.559670:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507283584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.560347:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507283712:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760756.594664:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972406 gen 3 00000008:00010000:1.0:1789760756.594750:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760756.594759:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760756.597073:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760756.597116:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760756.597158:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507283712:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760757.230934:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20881408 i_blocks 40967 00000020:00200000:1.0:1789760757.230940:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760757.230950:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760757.230985:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20881408, count: 4096 00000080:00200000:1.0:1789760757.230991:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20881408, bytes: 4096 00000080:00200000:1.0:1789760757.231003:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760757.231012:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760757.231014:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20881408, 20885504) 0 0 00000080:00200000:1.0:1789760757.231019:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760757.231038:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5098, 5098] 00020000:00200000:1.0:1789760757.231046:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20881408 chunk: [20881408, 20885504) 20885504, 4096 00020000:00200000:1.0:1789760757.231051:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760757.231073:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5098, 5098] 00000008:00010000:1.0:1789760757.231092:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760757.231098:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760757.231109:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20881408, 20885504) append 0 append lockpos 0 00000080:00200000:1.0:1789760757.231123:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20881408=0x13ea000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760757.231183:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760757.231186:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20885504 00000020:00010000:1.0:1789760757.231194:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760757.231198:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760757.231201:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54176c8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760757.231205:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760757.231208:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760757.231216:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760757.231222:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760757.231239:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760757.231572:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20881408<>20885504 00000100:00100000:1.0:1789760757.269064:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972418 gen 3 00000008:00010000:1.0:1789760757.269117:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:1.0:1789760757.269126:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.269137:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.269164:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:1.0:1789760757.269208:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507285248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.271360:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760757.271427:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507285376:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760757.304060:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760757.304075:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.304088:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.304305:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760757.304343:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507285376:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760757.304991:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20889600 i_blocks 40967 00000020:00200000:1.0:1789760757.304997:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760757.305008:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760757.305043:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20889600, count: 4096 00000080:00200000:1.0:1789760757.305052:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20889600, bytes: 4096 00000080:00200000:1.0:1789760757.305063:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760757.305072:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760757.305074:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20889600, 20893696) 0 0 00000080:00200000:1.0:1789760757.305079:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760757.305095:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5100, 5100] 00020000:00200000:1.0:1789760757.305105:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20889600 chunk: [20889600, 20893696) 20893696, 4096 00020000:00200000:1.0:1789760757.305110:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760757.305136:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5100, 5100] 00000008:00010000:1.0:1789760757.305156:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760757.305161:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760757.305173:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20889600, 20893696) append 0 append lockpos 0 00000080:00200000:1.0:1789760757.305187:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20889600=0x13ec000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760757.306146:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760757.306154:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20893696 00000020:00010000:1.0:1789760757.306166:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760757.306171:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760757.306174:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760757.306178:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760757.306181:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760757.306192:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760757.306199:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760757.306273:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760757.306416:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 20889600<>20893696 00000100:00100000:1.0:1789760757.306476:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760757.306499:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507285504:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760757.342780:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5100, 5100] 00000080:00200000:1.0:1789760757.342798:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760757.342804:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760757.342820:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 20893696 i_blocks 40967 00000020:00200000:1.0:1789760757.342824:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760757.342835:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760757.342877:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 20893696, count: 4096 00000080:00200000:1.0:1789760757.342884:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 20893696, bytes: 4096 00000080:00200000:1.0:1789760757.342894:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760757.342903:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760757.342906:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [20893696, 20897792) 0 0 00000080:00200000:1.0:1789760757.342910:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760757.342927:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5101, 5101] 00020000:00200000:1.0:1789760757.342936:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 20893696 chunk: [20893696, 20897792) 20897792, 4096 00020000:00200000:1.0:1789760757.342941:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760757.342959:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5101, 5101] 00000008:00010000:1.0:1789760757.342977:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760757.342983:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760757.342994:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [20893696, 20897792) append 0 append lockpos 0 00000080:00200000:1.0:1789760757.343008:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=20893696=0x13ed000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760757.343067:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760757.343070:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 20897792 00000020:00010000:1.0:1789760757.343079:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760757.343082:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760757.343085:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760757.343088:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760757.343091:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760757.343099:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760757.343105:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760757.343120:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760757.343675:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 20893696<>20897792 00000100:00100000:1.0:1789760757.409942:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972421 gen 3 00000008:00010000:1.0:1789760757.410051:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760757.410060:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.410071:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.410132:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760757.410168:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507285632:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.412849:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507285760:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.464992:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972422 gen 3 00000008:00010000:1.0:1789760757.465027:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760757.465033:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.465043:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.465100:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760757.465124:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507285760:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.465838:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507285888:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.505097:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972423 gen 3 00000008:00010000:1.0:1789760757.505132:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760757.505138:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.505148:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.505167:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760757.505190:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507285888:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.505820:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507286656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.549447:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972424 gen 3 00000008:00010000:1.0:1789760757.549485:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760757.549491:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.549502:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.549569:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760757.549593:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507286656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.551136:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507286784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.601311:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972425 gen 3 00000008:00010000:1.0:1789760757.601528:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760757.601534:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.601544:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.601564:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760757.601586:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507286784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.648071:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287040:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.708647:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972427 gen 3 00000008:00010000:1.0:1789760757.708694:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760757.708700:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.708710:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.708873:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760757.708898:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287040:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.710127:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760757.710219:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287168:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.745203:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972428 gen 3 00000008:00010000:1.0:1789760757.745335:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760757.745342:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.745351:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.745434:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760757.745467:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287168:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.747798:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287296:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760757.813160:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760757.813171:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.813181:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.813197:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760757.813221:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287296:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.813874:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507287424:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.852037:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972430 gen 3 00000008:00010000:1.0:1789760757.852083:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760757.852089:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.852097:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.852116:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760757.852148:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507287424:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.852774:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287552:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.903609:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972431 gen 3 00000008:00010000:1.0:1789760757.903647:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760757.903653:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.903661:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.903676:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760757.903700:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287552:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.904363:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507287680:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760757.941583:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760757.941593:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.941602:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.941656:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760757.941677:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507287680:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.942607:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287808:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760757.984983:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972432 gen 3 00000100:00100000:1.0:1789760757.985011:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b8e00 x1876699507287808/t4294972433(4294972433) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760773 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760757.985150:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760757.985156:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760757.985165:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760757.985244:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760757.985265:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507287808:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.059302:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288064:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.125790:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972435 gen 3 00000008:00010000:1.0:1789760758.126240:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760758.126247:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760758.126257:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760758.126316:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760758.126346:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288064:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.132672:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760758.132691:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.166715:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972436 gen 3 00000008:00010000:1.0:1789760758.166763:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760758.166770:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760758.166779:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760758.166822:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760758.166848:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.167563:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.194937:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972437 gen 3 00000008:00010000:1.0:1789760758.194993:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760758.195002:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760758.195014:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760758.195042:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760758.195074:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.242327:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760758.242355:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.277595:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972439 gen 3 00000008:00010000:1.0:1789760758.277647:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760758.277655:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760758.277669:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760758.277734:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760758.277766:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507288576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.314300:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972440 gen 3 00000008:00010000:1.0:1789760758.314363:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760758.314371:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760758.314422:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760758.314444:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760758.314472:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507288704:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760758.694109:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5128, 5128] 00000080:00200000:1.0:1789760758.694127:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760758.694133:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760758.694149:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21008384 i_blocks 43015 00000020:00200000:1.0:1789760758.694153:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760758.694164:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760758.694195:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21008384, count: 4096 00000080:00200000:1.0:1789760758.694202:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21008384, bytes: 4096 00000080:00200000:1.0:1789760758.694211:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760758.694220:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760758.694223:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21008384, 21012480) 0 0 00000080:00200000:1.0:1789760758.694228:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760758.694246:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5129, 5129] 00020000:00200000:1.0:1789760758.694254:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21008384 chunk: [21008384, 21012480) 21012480, 4096 00020000:00200000:1.0:1789760758.694259:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760758.694280:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5129, 5129] 00000008:00010000:1.0:1789760758.694297:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760758.694315:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760758.694327:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21008384, 21012480) append 0 append lockpos 0 00000080:00200000:1.0:1789760758.694341:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21008384=0x1409000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760758.694432:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760758.694435:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21012480 00000020:00010000:1.0:1789760758.694443:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760758.694447:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760758.694450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760758.694454:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760758.694458:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760758.694466:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760758.694473:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760758.694489:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760758.694589:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21008384<>21012480 00000100:00100000:1.0:1789760758.694637:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760758.694655:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507289856:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.735465:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972449 gen 3 00000008:00010000:1.0:1789760758.735520:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760758.735528:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760758.735541:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760758.735573:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760758.735607:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507289856:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760758.814040:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5131, 5131] 00000080:00200000:1.0:1789760758.814063:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760758.814069:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760758.814085:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21020672 i_blocks 43015 00000020:00200000:1.0:1789760758.814089:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760758.814099:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760758.814137:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21020672, count: 4096 00000080:00200000:1.0:1789760758.814144:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21020672, bytes: 4096 00000080:00200000:1.0:1789760758.814155:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760758.814164:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760758.814166:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21020672, 21024768) 0 0 00000080:00200000:1.0:1789760758.814171:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760758.814188:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5132, 5132] 00020000:00200000:1.0:1789760758.814197:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21020672 chunk: [21020672, 21024768) 21024768, 4096 00020000:00200000:1.0:1789760758.814203:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760758.814224:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5132, 5132] 00000008:00010000:1.0:1789760758.814243:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760758.814249:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760758.814261:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21020672, 21024768) append 0 append lockpos 0 00000080:00200000:1.0:1789760758.814277:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21020672=0x140c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760758.814594:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760758.814598:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21024768 00000020:00010000:1.0:1789760758.814608:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760758.814612:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760758.814615:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760758.814619:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760758.814622:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760758.814630:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760758.814637:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760758.814657:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760758.814769:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21020672<>21024768 00000100:00100000:1.0:1789760758.845251:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972452 gen 3 00000008:00010000:1.0:1789760758.845305:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760758.845314:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760758.845326:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760758.845352:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760758.845432:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507290240:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760758.954235:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760758.954255:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507290752:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.037007:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972456 gen 3 00000008:00010000:1.0:1789760759.037082:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760759.037093:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.037105:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.037182:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760759.037214:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507290752:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.040774:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507290880:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.071748:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972457 gen 3 00000008:00010000:1.0:1789760759.071802:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760759.071810:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.071822:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.071841:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760759.071884:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507290880:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.072707:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507291008:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.106524:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972458 gen 3 00000008:00010000:1.0:1789760759.106572:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760759.106580:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.106592:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.106734:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760759.106765:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507291008:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.300482:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507291648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.357169:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972463 gen 3 00000008:00010000:1.0:1789760759.357227:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760759.357236:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.357619:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.357642:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760759.357674:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507291648:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.359233:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760759.359268:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507291776:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.432126:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972464 gen 3 00000008:00010000:1.0:1789760759.432173:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760759.432181:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.432195:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.432265:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760759.432304:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507291776:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.433152:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507291904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.469354:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972465 gen 3 00000008:00010000:1.0:1789760759.469434:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760759.469443:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.469455:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.469477:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760759.469507:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507291904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.470421:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292032:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760759.509827:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760759.509842:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.509866:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.509983:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760759.510014:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292032:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.510673:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507292160:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.544570:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972467 gen 3 00000008:00010000:1.0:1789760759.544633:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760759.544642:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.544655:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.544679:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760759.544710:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507292160:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.545761:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292288:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760759.581050:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760759.581066:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.581080:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.581103:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760759.581134:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292288:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.581870:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760759.582011:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292416:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.616674:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972468 gen 3 00000100:00100000:1.0:1789760759.616716:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b9c00 x1876699507292416/t4294972469(4294972469) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760775 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760759.616938:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760759.616946:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.616960:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.616982:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760759.617009:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292416:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.619271:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760759.619317:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507292544:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.709459:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972470 gen 3 00000008:00010000:1.0:1789760759.709525:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760759.709533:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.709545:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.709572:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760759.709602:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507292544:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.710549:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760759.710571:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292672:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.789827:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972471 gen 3 00000008:00010000:1.0:1789760759.789885:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760759.789893:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.789905:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.789924:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760759.789964:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292672:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.790883:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760759.790902:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507292800:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.844252:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972472 gen 3 00000008:00010000:1.0:1789760759.844304:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760759.844313:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.844327:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.844351:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760759.844433:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507292800:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.845573:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760759.845597:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292928:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.889877:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972473 gen 3 00000008:00010000:1.0:1789760759.889931:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760759.889940:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.889953:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.889975:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760759.890006:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507292928:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.890717:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760759.890735:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507293056:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.932699:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972474 gen 3 00000008:00010000:1.0:1789760759.932758:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760759.932766:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.932779:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.932835:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760759.932877:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507293056:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.933735:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507293184:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.968644:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972475 gen 3 00000008:00010000:1.0:1789760759.968702:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760759.968712:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760759.968726:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760759.968749:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760759.969065:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507293184:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760759.969726:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507293312:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.034664:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972476 gen 3 00000008:00010000:1.0:1789760760.034721:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760760.034731:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.034745:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.034768:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760760.036062:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507293312:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.036872:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760760.037058:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507293440:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.076942:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972477 gen 3 00000008:00010000:1.0:1789760760.076995:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760760.077004:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.077016:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.077082:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760760.077116:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507293440:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.077819:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507293568:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.104014:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972478 gen 3 00000008:00010000:1.0:1789760760.104062:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760760.104070:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.104188:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.104296:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760760.105451:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507293568:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760760.105502:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000008:00100000:1.0:1789760760.105608:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21131264<>21135360 00000100:00100000:1.0:1789760760.105654:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507293696:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.137067:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972479 gen 3 00000008:00010000:1.0:1789760760.137115:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000020:00200000:1.0:1789760760.137123:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.137135:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.137215:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000100:00100000:1.0:1789760760.137246:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507293696:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760760.191989:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5160, 5160] 00000080:00200000:1.0:1789760760.192009:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760760.192015:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760760.192032:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21139456 i_blocks 43015 00000020:00200000:1.0:1789760760.192036:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760760.192047:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760760.192085:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21139456, count: 4096 00000080:00200000:1.0:1789760760.192093:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21139456, bytes: 4096 00000080:00200000:1.0:1789760760.192106:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760760.192115:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760760.192118:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21139456, 21143552) 0 0 00000080:00200000:1.0:1789760760.192122:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760760.192139:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5161, 5161] 00020000:00200000:1.0:1789760760.192158:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21139456 chunk: [21139456, 21143552) 21143552, 4096 00020000:00200000:1.0:1789760760.192165:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760760.382624:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5164, 5164] 00000080:00200000:1.0:1789760760.382644:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760760.382650:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760760.382665:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21155840 i_blocks 43015 00000020:00200000:1.0:1789760760.382670:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760760.382682:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760760.382716:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21155840, count: 4096 00000080:00200000:1.0:1789760760.382724:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21155840, bytes: 4096 00000080:00200000:1.0:1789760760.382737:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760760.382745:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760760.382748:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21155840, 21159936) 0 0 00000080:00200000:1.0:1789760760.382752:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760760.382768:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5165, 5165] 00020000:00200000:1.0:1789760760.382777:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21155840 chunk: [21155840, 21159936) 21159936, 4096 00020000:00200000:1.0:1789760760.382782:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760760.382802:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5165, 5165] 00000008:00010000:1.0:1789760760.382821:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760760.382827:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760760.382840:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21155840, 21159936) append 0 append lockpos 0 00000080:00200000:1.0:1789760760.382864:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21155840=0x142d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760760.384259:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760760.384268:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21159936 00000020:00010000:1.0:1789760760.384280:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760760.384285:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760760.384288:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760760.384291:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760760.384294:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760760.384305:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760760.384314:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760760.384346:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760760.384665:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21155840<>21159936 00000100:00100000:1.0:1789760760.384713:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760760.384740:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507294464:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.449065:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972485 gen 3 00000008:00010000:1.0:1789760760.449119:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760760.449127:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.449139:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.449168:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760760.449200:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507294464:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760760.698043:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5170, 5170] 00000080:00200000:1.0:1789760760.698061:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760760.698067:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760760.698084:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21180416 i_blocks 43015 00000020:00200000:1.0:1789760760.698088:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760760.698097:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760760.698133:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21180416, count: 4096 00000080:00200000:1.0:1789760760.698141:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21180416, bytes: 4096 00000080:00200000:1.0:1789760760.698150:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760760.698159:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760760.698161:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21180416, 21184512) 0 0 00000080:00200000:1.0:1789760760.698166:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760760.698183:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5171, 5171] 00020000:00200000:1.0:1789760760.698192:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21180416 chunk: [21180416, 21184512) 21184512, 4096 00020000:00200000:1.0:1789760760.698197:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760760.698240:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5171, 5171] 00000008:00010000:1.0:1789760760.698259:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760760.698264:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760760.698276:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21180416, 21184512) append 0 append lockpos 0 00000080:00200000:1.0:1789760760.698291:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21180416=0x1433000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760760.698348:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760760.698351:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21184512 00000020:00010000:1.0:1789760760.698360:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760760.698363:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760760.698366:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760760.698370:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760760.698373:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760760.698420:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760760.698427:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760760.698442:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760760.698721:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21180416<>21184512 00000008:00010000:1.0:1789760760.807030:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760760.807041:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.807050:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.807216:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760760.807241:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507295232:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.811746:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507295360:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.869639:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972491 gen 3 00000100:00100000:1.0:1789760760.869669:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b9180 x1876699507295360/t4294972492(4294972492) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760776 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760760.869702:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760760.869707:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.869715:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.869733:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760760.869752:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507295360:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.871327:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507295488:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760760.924423:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972493 gen 3 00000008:00010000:1.0:1789760760.924488:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760760.924497:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760760.924508:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760760.924593:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760760.924620:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507295488:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760761.392266:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5182, 5182] 00000080:00200000:1.0:1789760761.392342:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760761.392348:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760761.392365:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21229568 i_blocks 43015 00000020:00200000:1.0:1789760761.392369:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760761.392411:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760761.392447:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21229568, count: 4096 00000080:00200000:1.0:1789760761.392454:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21229568, bytes: 4096 00000080:00200000:1.0:1789760761.392464:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760761.392473:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760761.392475:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21229568, 21233664) 0 0 00000080:00200000:1.0:1789760761.392479:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760761.392495:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5183, 5183] 00020000:00200000:1.0:1789760761.392503:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21229568 chunk: [21229568, 21233664) 21233664, 4096 00020000:00200000:1.0:1789760761.392507:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760761.392527:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5183, 5183] 00000008:00010000:1.0:1789760761.392545:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760761.392550:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760761.392560:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21229568, 21233664) append 0 append lockpos 0 00000080:00200000:1.0:1789760761.392575:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21229568=0x143f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760761.392630:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760761.392633:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21233664 00000020:00010000:1.0:1789760761.392641:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760761.392645:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760761.392648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760761.392651:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760761.392654:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760761.392662:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760761.392669:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760761.392687:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760761.392795:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21229568<>21233664 00000100:00100000:1.0:1789760761.392908:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760761.392927:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507296768:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.429961:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972503 gen 3 00000008:00010000:1.0:1789760761.430005:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760761.430012:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.430021:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.430042:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760761.430067:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507296768:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760761.499357:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5185, 5185] 00000080:00200000:1.0:1789760761.499374:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760761.499412:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760761.499428:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21241856 i_blocks 43015 00000020:00200000:1.0:1789760761.499432:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760761.499443:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760761.499479:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21241856, count: 4096 00000080:00200000:1.0:1789760761.499485:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21241856, bytes: 4096 00000080:00200000:1.0:1789760761.499496:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760761.499505:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760761.499507:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21241856, 21245952) 0 0 00000080:00200000:1.0:1789760761.499511:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760761.499529:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5186, 5186] 00020000:00200000:1.0:1789760761.499538:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21241856 chunk: [21241856, 21245952) 21245952, 4096 00020000:00200000:1.0:1789760761.499543:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760761.499563:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5186, 5186] 00000008:00010000:1.0:1789760761.499581:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760761.499586:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760761.499597:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21241856, 21245952) append 0 append lockpos 0 00000080:00200000:1.0:1789760761.499612:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21241856=0x1442000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760761.499667:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760761.499670:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21245952 00000020:00010000:1.0:1789760761.499678:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760761.499681:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760761.499684:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760761.499687:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760761.499690:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760761.499698:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760761.499705:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760761.499724:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760761.499829:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21241856<>21245952 00000100:00100000:1.0:1789760761.534750:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972506 gen 3 00000008:00010000:1.0:1789760761.534798:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760761.534806:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.534816:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.534845:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760761.534874:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507297152:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760761.566230:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5187, 5187] 00000080:00200000:1.0:1789760761.566246:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760761.566251:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760761.566265:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21250048 i_blocks 43015 00000020:00200000:1.0:1789760761.566268:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760761.566333:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760761.566370:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21250048, count: 4096 00000080:00200000:1.0:1789760761.566414:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21250048, bytes: 4096 00000080:00200000:1.0:1789760761.566426:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760761.566436:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760761.566438:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21250048, 21254144) 0 0 00000080:00200000:1.0:1789760761.566443:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760761.566461:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5188, 5188] 00020000:00200000:1.0:1789760761.566469:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21250048 chunk: [21250048, 21254144) 21254144, 4096 00020000:00200000:1.0:1789760761.566475:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760761.566494:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5188, 5188] 00000008:00010000:1.0:1789760761.566512:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760761.566518:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760761.566528:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21250048, 21254144) append 0 append lockpos 0 00000080:00200000:1.0:1789760761.566542:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21250048=0x1444000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760761.566598:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760761.566601:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21254144 00000020:00010000:1.0:1789760761.566609:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760761.566612:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760761.566616:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760761.566619:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760761.566622:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760761.566630:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760761.566637:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760761.566655:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760761.567032:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21250048<>21254144 00000100:00100000:1.0:1789760761.567086:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760761.567106:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507297408:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.607683:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972508 gen 3 00000008:00010000:1.0:1789760761.607737:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760761.607745:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.607756:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.607854:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760761.607902:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507297408:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760761.684503:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760761.684520:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.684533:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.684558:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760761.684590:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507297664:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.685441:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507297792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.730670:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972511 gen 3 00000008:00010000:1.0:1789760761.730730:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760761.730737:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.730748:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.730811:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760761.730849:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507297792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.731657:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507297920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.770975:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972512 gen 3 00000008:00010000:1.0:1789760761.771022:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760761.771031:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.771044:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.771066:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760761.771101:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507297920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.773110:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507298048:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.811735:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972513 gen 3 00000008:00010000:1.0:1789760761.811787:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760761.811796:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.811809:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.811830:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760761.811871:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507298048:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760761.851848:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972514 gen 3 00000008:00010000:1.0:1789760761.851979:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760761.851989:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760761.852002:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760761.852076:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760761.852102:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507298176:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760762.122806:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5198, 5198] 00000080:00200000:1.0:1789760762.122823:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760762.122838:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760762.122854:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21295104 i_blocks 43015 00000020:00200000:1.0:1789760762.122859:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760762.122869:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760762.122901:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21295104, count: 4096 00000080:00200000:1.0:1789760762.122908:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21295104, bytes: 4096 00000080:00200000:1.0:1789760762.122933:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760762.122943:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760762.122945:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21295104, 21299200) 0 0 00000080:00200000:1.0:1789760762.122949:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760762.122967:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5199, 5199] 00020000:00200000:1.0:1789760762.122976:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21295104 chunk: [21295104, 21299200) 21299200, 4096 00020000:00200000:1.0:1789760762.122981:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760762.123002:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5199, 5199] 00000008:00010000:1.0:1789760762.123021:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760762.123028:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760762.123039:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21295104, 21299200) append 0 append lockpos 0 00000080:00200000:1.0:1789760762.123054:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21295104=0x144f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760762.123114:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760762.123117:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21299200 00000020:00010000:1.0:1789760762.123127:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9218 (ffff8d25c8038ab8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760762.123130:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9218 (ffff8d25c8038ab8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760762.123133:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760762.123137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9218 (ffff8d25c8038ab8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760762.123140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038ab8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760762.123148:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760762.123155:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760762.123173:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760762.123288:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21295104<>21299200 00000100:00100000:1.0:1789760762.123502:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760762.123522:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507298816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.172075:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972519 gen 3 00000008:00010000:1.0:1789760762.172130:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760762.172139:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760762.172150:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760762.172477:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760762.172507:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507298816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.259532:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507299200:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.292087:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972522 gen 3 00000008:00010000:1.0:1789760762.292137:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760762.292144:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760762.292156:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760762.292234:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760762.292266:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507299200:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.380823:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507299584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.416904:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972525 gen 3 00000008:00010000:1.0:1789760762.416976:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760762.416984:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760762.416996:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760762.417060:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760762.417092:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507299584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.418405:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760762.418426:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507299712:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760762.460883:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760762.460899:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760762.460912:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760762.460931:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760762.460964:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507299712:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.461902:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507299840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.505104:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972527 gen 3 00000008:00010000:1.0:1789760762.505169:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760762.505179:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760762.505189:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760762.505213:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760762.505248:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507299840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.508255:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507299968:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760762.556241:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760762.556257:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760762.556271:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760762.556306:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760762.556345:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507299968:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760762.603439:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5209, 5209] 00000080:00200000:1.0:1789760762.603458:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760762.603465:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760762.603482:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21340160 i_blocks 43015 00000020:00200000:1.0:1789760762.603486:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760762.603497:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760762.603529:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21340160, count: 4096 00000080:00200000:1.0:1789760762.603536:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21340160, bytes: 4096 00000080:00200000:1.0:1789760762.603548:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760762.603557:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760762.603560:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21340160, 21344256) 0 0 00000080:00200000:1.0:1789760762.603564:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760762.603581:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5210, 5210] 00020000:00200000:1.0:1789760762.603590:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21340160 chunk: [21340160, 21344256) 21344256, 4096 00020000:00200000:1.0:1789760762.603594:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760762.603614:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5210, 5210] 00000008:00010000:1.0:1789760762.603633:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760762.603640:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760762.603651:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21340160, 21344256) append 0 append lockpos 0 00000080:00200000:1.0:1789760762.603664:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21340160=0x145a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760762.603729:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760762.603732:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21344256 00000020:00010000:1.0:1789760762.603740:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760762.603743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760762.603746:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760762.603749:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760762.603752:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760762.603760:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760762.603767:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760762.603785:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760762.605434:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21340160<>21344256 00000100:00100000:1.0:1789760762.605506:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760762.605533:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507300224:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.647883:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507300992:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.717435:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507301248:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760762.993595:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507302016:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760763.233024:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507302656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760763.406022:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760763.407841:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507303168:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760763.469807:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5228, 5228] 00000080:00200000:1.0:1789760763.469829:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760763.469834:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760763.469845:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21417984 i_blocks 43015 00000020:00200000:1.0:1789760763.469848:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760763.469857:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760763.469881:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21417984, count: 4096 00000080:00200000:1.0:1789760763.469887:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21417984, bytes: 4096 00000080:00200000:1.0:1789760763.469894:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760763.470112:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760763.470114:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21417984, 21422080) 0 0 00000080:00200000:1.0:1789760763.470118:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760763.470130:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5229, 5229] 00020000:00200000:1.0:1789760763.470138:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21417984 chunk: [21417984, 21422080) 21422080, 4096 00020000:00200000:1.0:1789760763.470142:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760763.470156:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5229, 5229] 00000008:00010000:1.0:1789760763.470170:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760763.470174:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760763.470182:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21417984, 21422080) append 0 append lockpos 0 00000080:00200000:1.0:1789760763.470192:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21417984=0x146d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760763.470236:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760763.470238:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21422080 00000020:00010000:1.0:1789760763.470245:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760763.470248:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.470251:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.470254:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760763.470256:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760763.470262:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760763.470268:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760763.470283:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760763.470358:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 21417984<>21422080 00000080:00200000:1.0:1789760763.531935:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5229, 5229] 00000080:00200000:1.0:1789760763.531947:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760763.531952:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760763.531963:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21422080 i_blocks 43015 00000020:00200000:1.0:1789760763.531967:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760763.531975:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760763.532002:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21422080, count: 4096 00000080:00200000:1.0:1789760763.532009:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21422080, bytes: 4096 00000080:00200000:1.0:1789760763.532017:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760763.532025:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760763.532027:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21422080, 21426176) 0 0 00000080:00200000:1.0:1789760763.532031:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760763.532044:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5230, 5230] 00020000:00200000:1.0:1789760763.532051:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21422080 chunk: [21422080, 21426176) 21426176, 4096 00020000:00200000:1.0:1789760763.532054:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760763.532080:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5230, 5230] 00000008:00010000:1.0:1789760763.532095:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760763.532099:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760763.532108:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21422080, 21426176) append 0 append lockpos 0 00000080:00200000:1.0:1789760763.532119:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21422080=0x146e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760763.532163:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760763.532165:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21426176 00000020:00010000:1.0:1789760763.532173:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760763.532176:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.532179:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.532182:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760763.532185:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760763.532191:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760763.532196:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760763.532209:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760763.532295:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 21422080<>21426176 00000080:00200000:1.0:1789760763.603802:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5230, 5230] 00000080:00200000:1.0:1789760763.603818:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760763.603836:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760763.603851:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21426176 i_blocks 43015 00000020:00200000:1.0:1789760763.603855:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760763.603865:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760763.603897:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21426176, count: 4096 00000080:00200000:1.0:1789760763.603905:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21426176, bytes: 4096 00000080:00200000:1.0:1789760763.603915:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760763.603924:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760763.603927:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21426176, 21430272) 0 0 00000080:00200000:1.0:1789760763.603931:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760763.603963:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5231, 5231] 00020000:00200000:1.0:1789760763.603972:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21426176 chunk: [21426176, 21430272) 21430272, 4096 00020000:00200000:1.0:1789760763.603976:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760763.603999:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5231, 5231] 00000008:00010000:1.0:1789760763.604018:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760763.604024:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760763.604034:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21426176, 21430272) append 0 append lockpos 0 00000080:00200000:1.0:1789760763.604047:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21426176=0x146f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760763.604098:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760763.604100:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21430272 00000020:00010000:1.0:1789760763.604108:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760763.604111:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.604113:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.604117:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760763.604120:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760763.604128:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760763.604136:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760763.604154:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760763.604256:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 21426176<>21430272 00000080:00200000:1.0:1789760763.681016:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5231, 5231] 00000080:00200000:1.0:1789760763.681033:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760763.681039:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760763.681054:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21430272 i_blocks 43015 00000020:00200000:1.0:1789760763.681058:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760763.681067:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760763.681157:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21430272, count: 4096 00000080:00200000:1.0:1789760763.681165:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21430272, bytes: 4096 00000080:00200000:1.0:1789760763.681177:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760763.681185:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760763.681188:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21430272, 21434368) 0 0 00000080:00200000:1.0:1789760763.681192:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760763.681208:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5232, 5232] 00020000:00200000:1.0:1789760763.681216:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21430272 chunk: [21430272, 21434368) 21434368, 4096 00020000:00200000:1.0:1789760763.681221:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760763.681241:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5232, 5232] 00000008:00010000:1.0:1789760763.681425:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760763.681432:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760763.681445:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21430272, 21434368) append 0 append lockpos 0 00000080:00200000:1.0:1789760763.681460:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21430272=0x1470000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760763.681524:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760763.681527:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21434368 00000020:00010000:1.0:1789760763.681537:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760763.681540:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.681544:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.681547:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760763.681550:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760763.681558:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760763.681565:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760763.681585:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760763.681689:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 21430272<>21434368 00000080:00200000:1.0:1789760763.792781:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5232, 5232] 00000080:00200000:1.0:1789760763.792799:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760763.792804:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760763.792829:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21434368 i_blocks 43015 00000020:00200000:1.0:1789760763.792832:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760763.792843:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760763.792878:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21434368, count: 4096 00000080:00200000:1.0:1789760763.792885:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21434368, bytes: 4096 00000080:00200000:1.0:1789760763.792898:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760763.792907:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760763.792909:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21434368, 21438464) 0 0 00000080:00200000:1.0:1789760763.792914:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760763.792930:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5233, 5233] 00020000:00200000:1.0:1789760763.792938:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21434368 chunk: [21434368, 21438464) 21438464, 4096 00020000:00200000:1.0:1789760763.792943:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760763.792964:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5233, 5233] 00000008:00010000:1.0:1789760763.792983:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760763.792991:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760763.793003:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21434368, 21438464) append 0 append lockpos 0 00000080:00200000:1.0:1789760763.793020:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21434368=0x1471000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760763.793078:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760763.793081:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21438464 00000020:00010000:1.0:1789760763.793090:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760763.793093:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.793096:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.793099:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760763.793101:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760763.793108:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760763.793116:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760763.793132:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760763.793234:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 21434368<>21438464 00000100:00100000:1.0:1789760763.793290:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760763.793307:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507303808:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760763.883406:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5233, 5233] 00000080:00200000:1.0:1789760763.883425:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760763.883431:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760763.883447:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21438464 i_blocks 43015 00000020:00200000:1.0:1789760763.883451:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760763.883462:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760763.883509:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21438464, count: 4096 00000080:00200000:1.0:1789760763.883518:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21438464, bytes: 4096 00000080:00200000:1.0:1789760763.883532:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760763.883542:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760763.883544:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21438464, 21442560) 0 0 00000080:00200000:1.0:1789760763.883549:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760763.883566:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5234, 5234] 00020000:00200000:1.0:1789760763.883575:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21438464 chunk: [21438464, 21442560) 21442560, 4096 00020000:00200000:1.0:1789760763.883579:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760763.883599:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5234, 5234] 00000008:00010000:1.0:1789760763.883629:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760763.883637:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760763.883649:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21438464, 21442560) append 0 append lockpos 0 00000080:00200000:1.0:1789760763.883665:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21438464=0x1472000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760763.883727:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760763.883730:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21442560 00000020:00010000:1.0:1789760763.883739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760763.883743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.883746:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.883751:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760763.883754:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760763.883763:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760763.883770:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760763.883788:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760763.883906:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 21438464<>21442560 00000080:00200000:1.0:1789760763.964966:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5234, 5234] 00000080:00200000:1.0:1789760763.964984:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760763.964990:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760763.965007:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21442560 i_blocks 43015 00000020:00200000:1.0:1789760763.965012:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760763.965021:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760763.965056:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21442560, count: 4096 00000080:00200000:1.0:1789760763.965064:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21442560, bytes: 4096 00000080:00200000:1.0:1789760763.965075:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760763.965084:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760763.965087:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21442560, 21446656) 0 0 00000080:00200000:1.0:1789760763.965092:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760763.965109:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5235, 5235] 00020000:00200000:1.0:1789760763.965118:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21442560 chunk: [21442560, 21446656) 21446656, 4096 00020000:00200000:1.0:1789760763.965123:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760763.965144:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5235, 5235] 00000008:00010000:1.0:1789760763.965163:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760763.965170:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760763.965181:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21442560, 21446656) append 0 append lockpos 0 00000080:00200000:1.0:1789760763.965195:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21442560=0x1473000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760763.965253:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760763.965256:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21446656 00000020:00010000:1.0:1789760763.965275:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760763.965279:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.965283:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760763.965287:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760763.965290:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760763.965299:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760763.965306:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760763.965321:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760763.965472:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 21442560<>21446656 00000080:00200000:1.0:1789760764.014153:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5235, 5235] 00000080:00200000:1.0:1789760764.014171:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.014177:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.014194:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21446656 i_blocks 43015 00000020:00200000:1.0:1789760764.014197:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.014207:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.014241:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21446656, count: 4096 00000080:00200000:1.0:1789760764.014249:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21446656, bytes: 4096 00000080:00200000:1.0:1789760764.014259:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.014268:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.014270:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21446656, 21450752) 0 0 00000080:00200000:1.0:1789760764.014275:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.014292:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5236, 5236] 00020000:00200000:1.0:1789760764.014301:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21446656 chunk: [21446656, 21450752) 21450752, 4096 00020000:00200000:1.0:1789760764.014306:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.014326:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5236, 5236] 00000008:00010000:1.0:1789760764.014341:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.014346:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.014357:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21446656, 21450752) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.014372:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21446656=0x1474000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.014453:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.014456:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21450752 00000020:00010000:1.0:1789760764.014464:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.014468:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.014471:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.014474:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.014477:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.014485:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.014492:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.084099:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5236, 5236] 00000080:00200000:1.0:1789760764.084122:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.084129:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.084145:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21450752 i_blocks 43015 00000020:00200000:1.0:1789760764.084149:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.084217:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.084261:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21450752, count: 4096 00000080:00200000:1.0:1789760764.084269:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21450752, bytes: 4096 00000080:00200000:1.0:1789760764.084280:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.084289:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.084291:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21450752, 21454848) 0 0 00000080:00200000:1.0:1789760764.084297:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.084315:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5237, 5237] 00020000:00200000:1.0:1789760764.084324:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21450752 chunk: [21450752, 21454848) 21454848, 4096 00020000:00200000:1.0:1789760764.084337:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.084359:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5237, 5237] 00000008:00010000:1.0:1789760764.084411:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.084418:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.084430:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21450752, 21454848) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.084446:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21450752=0x1475000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.084522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.084526:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21454848 00000020:00010000:1.0:1789760764.084535:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.084539:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.084542:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.084546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.084549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.084557:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.084564:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.084583:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760764.084690:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 21450752<>21454848 00000080:00200000:1.0:1789760764.171708:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5237, 5237] 00000080:00200000:1.0:1789760764.171724:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.171730:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.171747:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21454848 i_blocks 43015 00000020:00200000:1.0:1789760764.171751:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.171761:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.171795:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21454848, count: 4096 00000080:00200000:1.0:1789760764.171802:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21454848, bytes: 4096 00000080:00200000:1.0:1789760764.171828:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.171837:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.171840:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21454848, 21458944) 0 0 00000080:00200000:1.0:1789760764.171845:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.171873:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5238, 5238] 00020000:00200000:1.0:1789760764.171881:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21454848 chunk: [21454848, 21458944) 21458944, 4096 00020000:00200000:1.0:1789760764.171886:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.172335:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5238, 5238] 00000008:00010000:1.0:1789760764.172355:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.172361:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.172373:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21454848, 21458944) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.172412:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21454848=0x1476000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.172472:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.172475:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21458944 00000020:00010000:1.0:1789760764.172484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.172488:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.172491:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.172495:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.172498:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.172506:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.172513:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.172536:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760764.172653:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 21454848<>21458944 00000100:00100000:1.0:1789760764.172731:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507304448:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760764.277639:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5238, 5238] 00000080:00200000:1.0:1789760764.277672:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.277679:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.277698:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21458944 i_blocks 43015 00000020:00200000:1.0:1789760764.277704:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.277720:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.277828:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21458944, count: 4096 00000080:00200000:1.0:1789760764.277838:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21458944, bytes: 4096 00000080:00200000:1.0:1789760764.277854:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.277863:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.277866:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21458944, 21463040) 0 0 00000080:00200000:1.0:1789760764.277871:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.277888:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5239, 5239] 00020000:00200000:1.0:1789760764.277898:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21458944 chunk: [21458944, 21463040) 21463040, 4096 00020000:00200000:1.0:1789760764.277903:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.277925:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5239, 5239] 00000008:00010000:1.0:1789760764.277946:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.277952:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.277965:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21458944, 21463040) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.277980:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21458944=0x1477000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.278057:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.278061:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21463040 00000020:00010000:1.0:1789760764.278070:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.278073:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.278076:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.278080:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.278083:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.278093:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.278100:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.278135:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760764.278244:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 21458944<>21463040 00000080:00200000:1.0:1789760764.310353:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5239, 5239] 00000080:00200000:1.0:1789760764.310370:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.310403:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.310420:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21463040 i_blocks 43015 00000020:00200000:1.0:1789760764.310425:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.373670:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5240, 5240] 00000080:00200000:1.0:1789760764.373688:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.373695:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.373713:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21467136 i_blocks 43015 00000020:00200000:1.0:1789760764.373717:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.373727:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.373772:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21467136, count: 4096 00000080:00200000:1.0:1789760764.373781:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21467136, bytes: 4096 00000080:00200000:1.0:1789760764.373790:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.373799:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.373802:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21467136, 21471232) 0 0 00000080:00200000:1.0:1789760764.373806:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.373832:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5241, 5241] 00020000:00200000:1.0:1789760764.373840:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21467136 chunk: [21467136, 21471232) 21471232, 4096 00020000:00200000:1.0:1789760764.373847:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.373874:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5241, 5241] 00000008:00010000:1.0:1789760764.373893:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.373899:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.373910:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21467136, 21471232) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.373923:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21467136=0x1479000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.374139:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.374142:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21471232 00000020:00010000:1.0:1789760764.374151:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.374154:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.374157:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.374160:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.374163:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.374170:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.374176:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.420088:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5241, 5241] 00000080:00200000:1.0:1789760764.420106:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.420112:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.420129:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21471232 i_blocks 43015 00000020:00200000:1.0:1789760764.420132:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.420144:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.420178:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21471232, count: 4096 00000080:00200000:1.0:1789760764.420187:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21471232, bytes: 4096 00000080:00200000:1.0:1789760764.420198:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.420207:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.420209:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21471232, 21475328) 0 0 00000080:00200000:1.0:1789760764.420214:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.420231:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5242, 5242] 00020000:00200000:1.0:1789760764.420239:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21471232 chunk: [21471232, 21475328) 21475328, 4096 00020000:00200000:1.0:1789760764.420245:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.420338:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5242, 5242] 00000008:00010000:1.0:1789760764.420360:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.420367:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.420614:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21471232, 21475328) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.420630:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21471232=0x147a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.420683:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.420686:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21475328 00000020:00010000:1.0:1789760764.420695:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.420698:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.420701:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.420705:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.420708:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.420716:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.420723:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:1.0:1789760764.421651:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507304960:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760764.465604:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5242, 5242] 00000080:00200000:1.0:1789760764.465620:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.465627:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.465642:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21475328 i_blocks 43015 00000020:00200000:1.0:1789760764.465646:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.465656:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.465688:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21475328, count: 4096 00000080:00200000:1.0:1789760764.465695:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21475328, bytes: 4096 00000080:00200000:1.0:1789760764.465705:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.465714:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.465716:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21475328, 21479424) 0 0 00000080:00200000:1.0:1789760764.465720:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.465796:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5243, 5243] 00020000:00200000:1.0:1789760764.465805:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21475328 chunk: [21475328, 21479424) 21479424, 4096 00020000:00200000:1.0:1789760764.465809:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.465839:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5243, 5243] 00000008:00010000:1.0:1789760764.465861:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.465867:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.465877:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21475328, 21479424) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.465891:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21475328=0x147b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.466127:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.466130:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21479424 00000020:00010000:1.0:1789760764.466138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.466141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.466144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.466147:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.466150:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.466158:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.466165:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.466183:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760764.466356:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 21475328<>21479424 00000100:00100000:1.0:1789760764.466478:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760764.466497:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507305088:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760764.499203:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5243, 5243] 00000080:00200000:1.0:1789760764.499221:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.499227:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.499242:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21479424 i_blocks 43015 00000020:00200000:1.0:1789760764.499245:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.499255:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.499288:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21479424, count: 4096 00000080:00200000:1.0:1789760764.499295:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21479424, bytes: 4096 00000080:00200000:1.0:1789760764.499304:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.499313:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.499315:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21479424, 21483520) 0 0 00000080:00200000:1.0:1789760764.499320:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.499337:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5244, 5244] 00020000:00200000:1.0:1789760764.499346:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21479424 chunk: [21479424, 21483520) 21483520, 4096 00020000:00200000:1.0:1789760764.499361:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.499423:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5244, 5244] 00000008:00010000:1.0:1789760764.499442:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.499448:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.499460:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21479424, 21483520) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.499473:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21479424=0x147c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.499530:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.499533:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21483520 00000020:00010000:1.0:1789760764.499542:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.499545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.499548:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.499552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.499554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.499563:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.499570:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.499588:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760764.499688:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 21479424<>21483520 00000080:00200000:1.0:1789760764.539226:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5244, 5244] 00000080:00200000:1.0:1789760764.539345:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.539352:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.539369:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21483520 i_blocks 43015 00000020:00200000:1.0:1789760764.539373:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.539402:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.539439:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21483520, count: 4096 00000080:00200000:1.0:1789760764.539446:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21483520, bytes: 4096 00000080:00200000:1.0:1789760764.539460:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.539469:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.539471:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21483520, 21487616) 0 0 00000080:00200000:1.0:1789760764.539476:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.539491:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5245, 5245] 00020000:00200000:1.0:1789760764.539500:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21483520 chunk: [21483520, 21487616) 21487616, 4096 00020000:00200000:1.0:1789760764.539504:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.539525:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5245, 5245] 00000008:00010000:1.0:1789760764.539543:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.539550:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.539561:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21483520, 21487616) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.539577:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21483520=0x147d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.539631:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.539634:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21487616 00000020:00010000:1.0:1789760764.539642:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.539645:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.539648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.539653:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.539655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.539663:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.539670:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.539688:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760764.539798:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 21483520<>21487616 00000100:00100000:1.0:1789760764.540201:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760764.540220:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507305344:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760764.622773:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5245, 5245] 00000080:00200000:1.0:1789760764.622792:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.622799:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.622826:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21487616 i_blocks 43015 00000020:00200000:1.0:1789760764.622830:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.622839:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.622877:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21487616, count: 4096 00000080:00200000:1.0:1789760764.622884:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21487616, bytes: 4096 00000080:00200000:1.0:1789760764.622896:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.622904:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.622908:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21487616, 21491712) 0 0 00000080:00200000:1.0:1789760764.622912:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.622929:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5246, 5246] 00020000:00200000:1.0:1789760764.622937:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21487616 chunk: [21487616, 21491712) 21491712, 4096 00020000:00200000:1.0:1789760764.622941:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.622962:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5246, 5246] 00000008:00010000:1.0:1789760764.622979:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.622986:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.622998:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21487616, 21491712) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.623013:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21487616=0x147e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.623072:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.623075:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21491712 00000020:00010000:1.0:1789760764.623084:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.623087:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.623091:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.623095:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.623098:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.623115:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.623123:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.623140:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760764.623259:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 21487616<>21491712 00000080:00200000:1.0:1789760764.673694:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5246, 5246] 00000080:00200000:1.0:1789760764.673711:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.673717:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.673733:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21491712 i_blocks 43015 00000020:00200000:1.0:1789760764.673737:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.673747:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.673781:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21491712, count: 4096 00000080:00200000:1.0:1789760764.673787:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21491712, bytes: 4096 00000080:00200000:1.0:1789760764.673798:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.673807:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.673821:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21491712, 21495808) 0 0 00000080:00200000:1.0:1789760764.673826:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.673843:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5247, 5247] 00020000:00200000:1.0:1789760764.673860:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21491712 chunk: [21491712, 21495808) 21495808, 4096 00020000:00200000:1.0:1789760764.673865:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.673885:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5247, 5247] 00000008:00010000:1.0:1789760764.673911:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.673917:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.673929:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21491712, 21495808) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.673943:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21491712=0x147f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.673997:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.674000:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21495808 00000020:00010000:1.0:1789760764.674009:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.674013:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.674016:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.674019:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.674022:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.674030:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.674037:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.674057:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760764.674236:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 21491712<>21495808 00000080:00200000:1.0:1789760764.732047:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5247, 5247] 00000080:00200000:1.0:1789760764.732073:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.732080:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.732100:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21495808 i_blocks 43015 00000020:00200000:1.0:1789760764.732106:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.732121:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.732153:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21495808, count: 4096 00000080:00200000:1.0:1789760764.732161:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21495808, bytes: 4096 00000080:00200000:1.0:1789760764.732175:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.732184:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.732187:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21495808, 21499904) 0 0 00000080:00200000:1.0:1789760764.732191:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.732206:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5248, 5248] 00020000:00200000:1.0:1789760764.732214:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21495808 chunk: [21495808, 21499904) 21499904, 4096 00020000:00200000:1.0:1789760764.732219:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.732240:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5248, 5248] 00000008:00010000:1.0:1789760764.732258:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.732264:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.732275:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21495808, 21499904) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.732290:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21495808=0x1480000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.732347:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.732350:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21499904 00000020:00010000:1.0:1789760764.732359:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.732363:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.732365:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.732369:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.732372:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.732491:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.732499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.732614:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760764.732734:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 21495808<>21499904 00000100:00100000:1.0:1789760764.732803:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507305728:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760764.789203:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5248, 5248] 00000080:00200000:1.0:1789760764.789220:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760764.789226:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760764.789242:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21499904 i_blocks 43015 00000020:00200000:1.0:1789760764.789246:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760764.789319:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760764.789434:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21499904, count: 4096 00000080:00200000:1.0:1789760764.789444:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21499904, bytes: 4096 00000080:00200000:1.0:1789760764.789456:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760764.789467:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760764.789470:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21499904, 21504000) 0 0 00000080:00200000:1.0:1789760764.789475:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760764.789492:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5249, 5249] 00020000:00200000:1.0:1789760764.789502:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21499904 chunk: [21499904, 21504000) 21504000, 4096 00020000:00200000:1.0:1789760764.789507:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760764.789528:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5249, 5249] 00000008:00010000:1.0:1789760764.789548:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760764.789554:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760764.789569:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21499904, 21504000) append 0 append lockpos 0 00000080:00200000:1.0:1789760764.789584:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21499904=0x1481000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760764.789651:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760764.789654:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21504000 00000020:00010000:1.0:1789760764.789663:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760764.789666:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.789669:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760764.789674:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760764.789677:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760764.789685:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760764.789693:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760764.789712:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760764.790542:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 21499904<>21504000 00000100:00100000:1.0:1789760764.790601:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507305856:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760764.894625:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507306112:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760765.064281:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5254, 5254] 00000080:00200000:1.0:1789760765.064299:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.064305:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.064321:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21524480 i_blocks 43015 00000020:00200000:1.0:1789760765.064325:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.064335:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.064370:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21524480, count: 4096 00000080:00200000:1.0:1789760765.064399:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21524480, bytes: 4096 00000080:00200000:1.0:1789760765.064411:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.064421:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.064423:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21524480, 21528576) 0 0 00000080:00200000:1.0:1789760765.064427:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.064442:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5255, 5255] 00020000:00200000:1.0:1789760765.064450:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21524480 chunk: [21524480, 21528576) 21528576, 4096 00020000:00200000:1.0:1789760765.064454:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.064474:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5255, 5255] 00000008:00010000:1.0:1789760765.064491:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.064498:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.064509:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21524480, 21528576) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.064522:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21524480=0x1487000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.064575:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.064578:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21528576 00000020:00010000:1.0:1789760765.064585:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.064589:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.064591:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.064595:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.064597:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.064604:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.064611:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.064628:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760765.064739:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 21524480<>21528576 00000080:00200000:1.0:1789760765.123886:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5255, 5255] 00000080:00200000:1.0:1789760765.123904:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.123911:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.123929:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21528576 i_blocks 43015 00000020:00200000:1.0:1789760765.123933:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.123942:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.123978:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21528576, count: 4096 00000080:00200000:1.0:1789760765.123986:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21528576, bytes: 4096 00000080:00200000:1.0:1789760765.123998:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.124008:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.124010:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21528576, 21532672) 0 0 00000080:00200000:1.0:1789760765.124014:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.124030:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5256, 5256] 00020000:00200000:1.0:1789760765.124039:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21528576 chunk: [21528576, 21532672) 21532672, 4096 00020000:00200000:1.0:1789760765.124044:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.124064:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5256, 5256] 00000008:00010000:1.0:1789760765.124083:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.124089:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.124101:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21528576, 21532672) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.124115:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21528576=0x1488000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.124177:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.124180:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21532672 00000020:00010000:1.0:1789760765.124189:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.124193:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.124197:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.124200:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.124203:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.124211:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.124218:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.124239:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760765.124354:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 21528576<>21532672 00000080:00200000:1.0:1789760765.359515:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5260, 5260] 00000080:00200000:1.0:1789760765.359534:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.359540:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.359557:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21549056 i_blocks 43015 00000020:00200000:1.0:1789760765.359561:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.359571:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.359606:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21549056, count: 4096 00000080:00200000:1.0:1789760765.359615:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21549056, bytes: 4096 00000080:00200000:1.0:1789760765.359625:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.359635:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.359637:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21549056, 21553152) 0 0 00000080:00200000:1.0:1789760765.359642:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.359659:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5261, 5261] 00020000:00200000:1.0:1789760765.359668:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21549056 chunk: [21549056, 21553152) 21553152, 4096 00020000:00200000:1.0:1789760765.359673:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.359694:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5261, 5261] 00000008:00010000:1.0:1789760765.359723:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.359729:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.359741:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21549056, 21553152) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.359756:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21549056=0x148d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.360404:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.360408:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21553152 00000020:00010000:1.0:1789760765.360418:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.360423:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.360426:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.360431:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.360434:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.360443:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.360450:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.360468:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760765.360580:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 21549056<>21553152 00000080:00200000:1.0:1789760765.398349:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5261, 5261] 00000080:00200000:1.0:1789760765.398369:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.398415:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.398433:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21553152 i_blocks 43015 00000020:00200000:1.0:1789760765.398438:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.398450:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.398567:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21553152, count: 4096 00000080:00200000:1.0:1789760765.398575:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21553152, bytes: 4096 00000080:00200000:1.0:1789760765.398588:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.398596:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.398599:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21553152, 21557248) 0 0 00000080:00200000:1.0:1789760765.398604:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.398621:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5262, 5262] 00020000:00200000:1.0:1789760765.398630:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21553152 chunk: [21553152, 21557248) 21557248, 4096 00020000:00200000:1.0:1789760765.398634:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.398658:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5262, 5262] 00000008:00010000:1.0:1789760765.398677:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.398684:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.398695:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21553152, 21557248) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.398709:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21553152=0x148e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.398771:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.398774:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21557248 00000020:00010000:1.0:1789760765.398783:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.398787:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.398789:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.398792:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.398795:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.398803:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.398830:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.398848:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760765.398955:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 21553152<>21557248 00000080:00200000:1.0:1789760765.436685:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5262, 5262] 00000080:00200000:1.0:1789760765.436701:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.436707:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.436724:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21557248 i_blocks 43015 00000020:00200000:1.0:1789760765.436729:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.436738:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.436772:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21557248, count: 4096 00000080:00200000:1.0:1789760765.436779:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21557248, bytes: 4096 00000080:00200000:1.0:1789760765.436788:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.436797:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.436800:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21557248, 21561344) 0 0 00000080:00200000:1.0:1789760765.436822:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.436840:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5263, 5263] 00020000:00200000:1.0:1789760765.436849:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21557248 chunk: [21557248, 21561344) 21561344, 4096 00020000:00200000:1.0:1789760765.436853:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.436871:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5263, 5263] 00000008:00010000:1.0:1789760765.436891:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.436897:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.436909:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21557248, 21561344) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.436923:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21557248=0x148f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.436979:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.436983:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21561344 00000020:00010000:1.0:1789760765.436991:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.436994:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.436997:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.437001:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.437003:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.437011:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.437017:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.437033:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760765.437136:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 21557248<>21561344 00000080:00200000:1.0:1789760765.572468:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5265, 5265] 00000080:00200000:1.0:1789760765.572484:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.572490:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.572506:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21569536 i_blocks 43015 00000020:00200000:1.0:1789760765.572511:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.572520:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.572551:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21569536, count: 4096 00000080:00200000:1.0:1789760765.572559:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21569536, bytes: 4096 00000080:00200000:1.0:1789760765.572571:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.572580:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.572582:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21569536, 21573632) 0 0 00000080:00200000:1.0:1789760765.572587:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.572602:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5266, 5266] 00020000:00200000:1.0:1789760765.572610:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21569536 chunk: [21569536, 21573632) 21573632, 4096 00020000:00200000:1.0:1789760765.572614:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.572634:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5266, 5266] 00000008:00010000:1.0:1789760765.572652:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.572659:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.572670:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21569536, 21573632) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.572684:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21569536=0x1492000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.572758:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.572761:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21573632 00000020:00010000:1.0:1789760765.572769:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.572772:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.572775:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.572779:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.572783:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.572790:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.572797:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.572825:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760765.572947:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21569536<>21573632 00000100:00100000:1.0:1789760765.572992:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760765.573009:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507308032:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760765.660748:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5267, 5267] 00000080:00200000:1.0:1789760765.660766:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.660772:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.660911:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21577728 i_blocks 43015 00000020:00200000:1.0:1789760765.660914:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.660925:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.660959:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21577728, count: 4096 00000080:00200000:1.0:1789760765.660966:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21577728, bytes: 4096 00000080:00200000:1.0:1789760765.660980:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.660989:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.660992:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21577728, 21581824) 0 0 00000080:00200000:1.0:1789760765.660996:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.661013:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5268, 5268] 00020000:00200000:1.0:1789760765.661022:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21577728 chunk: [21577728, 21581824) 21581824, 4096 00020000:00200000:1.0:1789760765.661026:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.661048:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5268, 5268] 00000008:00010000:1.0:1789760765.661066:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.661071:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.661082:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21577728, 21581824) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.661096:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21577728=0x1494000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.661150:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.661153:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21581824 00000020:00010000:1.0:1789760765.661162:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.661165:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.661168:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.661172:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.661175:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.661183:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.661190:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.661208:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760765.661588:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 21577728<>21581824 00000100:00100000:1.0:1789760765.792666:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507308672:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760765.963371:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5274, 5274] 00000080:00200000:1.0:1789760765.963418:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760765.963429:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760765.963445:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21606400 i_blocks 43015 00000020:00200000:1.0:1789760765.963449:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760765.963459:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760765.963494:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21606400, count: 4096 00000080:00200000:1.0:1789760765.963501:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21606400, bytes: 4096 00000080:00200000:1.0:1789760765.963512:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760765.963522:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760765.963524:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21606400, 21610496) 0 0 00000080:00200000:1.0:1789760765.963529:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760765.963545:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5275, 5275] 00020000:00200000:1.0:1789760765.963554:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21606400 chunk: [21606400, 21610496) 21610496, 4096 00020000:00200000:1.0:1789760765.963559:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760765.963579:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5275, 5275] 00000008:00010000:1.0:1789760765.963597:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760765.963604:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760765.963615:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21606400, 21610496) append 0 append lockpos 0 00000080:00200000:1.0:1789760765.963629:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21606400=0x149b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760765.963691:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760765.963694:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21610496 00000020:00010000:1.0:1789760765.963703:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760765.963707:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.963710:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760765.963714:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760765.963718:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760765.963726:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760765.963733:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760765.963750:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760765.963878:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 21606400<>21610496 00000100:00100000:1.0:1789760765.963919:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760765.963935:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507309184:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760766.015509:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760766.015933:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507309312:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760766.294463:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5282, 5282] 00000080:00200000:1.0:1789760766.294481:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.294487:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760766.294504:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21639168 i_blocks 43015 00000020:00200000:1.0:1789760766.294508:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760766.294519:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760766.294554:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21639168, count: 4096 00000080:00200000:1.0:1789760766.294562:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21639168, bytes: 4096 00000080:00200000:1.0:1789760766.294574:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760766.294583:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760766.294586:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21639168, 21643264) 0 0 00000080:00200000:1.0:1789760766.294590:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760766.294609:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5283, 5283] 00020000:00200000:1.0:1789760766.294618:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21639168 chunk: [21639168, 21643264) 21643264, 4096 00020000:00200000:1.0:1789760766.294622:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760766.294643:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5283, 5283] 00000008:00010000:1.0:1789760766.294664:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760766.294671:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760766.294682:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21639168, 21643264) append 0 append lockpos 0 00000080:00200000:1.0:1789760766.294697:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21639168=0x14a3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760766.294757:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760766.294760:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21643264 00000020:00010000:1.0:1789760766.294768:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760766.294772:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.294775:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.294780:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760766.294783:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760766.294791:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760766.294809:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760766.294827:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760766.294950:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 21639168<>21643264 00000100:00100000:1.0:1789760766.294995:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760766.295015:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507310208:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760766.340226:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5283, 5283] 00000080:00200000:1.0:1789760766.340242:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.340248:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760766.340264:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21643264 i_blocks 43015 00000020:00200000:1.0:1789760766.340267:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760766.340277:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760766.340312:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21643264, count: 4096 00000080:00200000:1.0:1789760766.340319:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21643264, bytes: 4096 00000080:00200000:1.0:1789760766.340329:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760766.340338:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760766.340340:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21643264, 21647360) 0 0 00000080:00200000:1.0:1789760766.340345:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760766.340362:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5284, 5284] 00020000:00200000:1.0:1789760766.340371:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21643264 chunk: [21643264, 21647360) 21647360, 4096 00020000:00200000:1.0:1789760766.340406:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760766.340427:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5284, 5284] 00000008:00010000:1.0:1789760766.340447:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760766.340452:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760766.340463:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21643264, 21647360) append 0 append lockpos 0 00000080:00200000:1.0:1789760766.340478:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21643264=0x14a4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760766.340532:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760766.340535:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21647360 00000020:00010000:1.0:1789760766.340544:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760766.340549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.340552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.340556:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760766.340559:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760766.340566:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760766.340574:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760766.340590:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760766.340682:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 21643264<>21647360 00000080:00200000:1.0:1789760766.396364:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5284, 5284] 00000080:00200000:1.0:1789760766.396574:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.396582:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760766.396601:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21647360 i_blocks 43015 00000020:00200000:1.0:1789760766.396606:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760766.396618:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760766.396658:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21647360, count: 4096 00000080:00200000:1.0:1789760766.396666:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21647360, bytes: 4096 00000080:00200000:1.0:1789760766.396679:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760766.396759:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760766.396763:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21647360, 21651456) 0 0 00000080:00200000:1.0:1789760766.396768:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760766.396787:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5285, 5285] 00020000:00200000:1.0:1789760766.396795:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21647360 chunk: [21647360, 21651456) 21651456, 4096 00020000:00200000:1.0:1789760766.396809:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760766.396832:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5285, 5285] 00000008:00010000:1.0:1789760766.396851:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760766.396857:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760766.396870:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21647360, 21651456) append 0 append lockpos 0 00000080:00200000:1.0:1789760766.396945:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21647360=0x14a5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760766.397022:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760766.397025:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21651456 00000020:00010000:1.0:1789760766.397034:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760766.397038:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.397041:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.397044:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760766.397047:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760766.397055:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760766.397062:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760766.397154:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760766.397339:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 21647360<>21651456 00000080:00200000:1.0:1789760766.463551:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5285, 5285] 00000080:00200000:1.0:1789760766.463568:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.463574:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760766.463591:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21651456 i_blocks 43015 00000020:00200000:1.0:1789760766.463594:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760766.463605:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760766.463643:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21651456, count: 4096 00000080:00200000:1.0:1789760766.463650:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21651456, bytes: 4096 00000080:00200000:1.0:1789760766.463663:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760766.463672:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760766.463675:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21651456, 21655552) 0 0 00000080:00200000:1.0:1789760766.463679:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760766.463696:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5286, 5286] 00020000:00200000:1.0:1789760766.463705:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21651456 chunk: [21651456, 21655552) 21655552, 4096 00020000:00200000:1.0:1789760766.463710:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760766.463730:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5286, 5286] 00000008:00010000:1.0:1789760766.463749:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760766.463756:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760766.463768:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21651456, 21655552) append 0 append lockpos 0 00000080:00200000:1.0:1789760766.463782:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21651456=0x14a6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760766.463851:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760766.463854:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21655552 00000020:00010000:1.0:1789760766.463862:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760766.463866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.463870:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.463874:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760766.463877:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760766.463886:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760766.463893:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760766.463909:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760766.464019:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 21651456<>21655552 00000080:00200000:1.0:1789760766.550434:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5286, 5286] 00000080:00200000:1.0:1789760766.550452:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.550459:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760766.550477:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21655552 i_blocks 43015 00000020:00200000:1.0:1789760766.550481:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760766.550492:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760766.550530:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21655552, count: 4096 00000080:00200000:1.0:1789760766.550537:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21655552, bytes: 4096 00000080:00200000:1.0:1789760766.550549:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760766.550558:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760766.550561:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21655552, 21659648) 0 0 00000080:00200000:1.0:1789760766.550566:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760766.550585:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5287, 5287] 00020000:00200000:1.0:1789760766.550594:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21655552 chunk: [21655552, 21659648) 21659648, 4096 00020000:00200000:1.0:1789760766.550598:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760766.550620:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5287, 5287] 00000008:00010000:1.0:1789760766.550639:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760766.550646:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760766.550658:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21655552, 21659648) append 0 append lockpos 0 00000080:00200000:1.0:1789760766.550672:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21655552=0x14a7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760766.550746:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760766.550750:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21659648 00000020:00010000:1.0:1789760766.550759:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760766.550763:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.550767:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.550771:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760766.550774:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760766.550782:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760766.550790:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760766.550820:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760766.550930:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 21655552<>21659648 00000080:00200000:1.0:1789760766.618352:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5287, 5287] 00000080:00200000:1.0:1789760766.618368:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.618375:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760766.618426:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21659648 i_blocks 43015 00000020:00200000:1.0:1789760766.618430:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760766.618441:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760766.618475:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21659648, count: 4096 00000080:00200000:1.0:1789760766.618483:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21659648, bytes: 4096 00000080:00200000:1.0:1789760766.618497:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760766.618508:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760766.618510:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21659648, 21663744) 0 0 00000080:00200000:1.0:1789760766.618515:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760766.618533:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5288, 5288] 00020000:00200000:1.0:1789760766.618542:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21659648 chunk: [21659648, 21663744) 21663744, 4096 00020000:00200000:1.0:1789760766.618547:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760766.618568:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5288, 5288] 00000008:00010000:1.0:1789760766.618587:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760766.618593:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760766.618605:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21659648, 21663744) append 0 append lockpos 0 00000080:00200000:1.0:1789760766.618619:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21659648=0x14a8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760766.618676:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760766.618679:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21663744 00000020:00010000:1.0:1789760766.618688:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760766.618693:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.618696:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.618700:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760766.618702:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760766.618710:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760766.618717:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760766.618734:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760766.618849:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 21659648<>21663744 00000080:00200000:1.0:1789760766.667686:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5288, 5288] 00000080:00200000:1.0:1789760766.667707:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.667713:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760766.667728:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21663744 i_blocks 43015 00000020:00200000:1.0:1789760766.667732:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760766.667744:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760766.667771:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21663744, count: 4096 00000080:00200000:1.0:1789760766.667777:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21663744, bytes: 4096 00000080:00200000:1.0:1789760766.667787:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760766.667794:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760766.667805:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21663744, 21667840) 0 0 00000080:00200000:1.0:1789760766.667809:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760766.667822:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5289, 5289] 00020000:00200000:1.0:1789760766.667828:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21663744 chunk: [21663744, 21667840) 21667840, 4096 00020000:00200000:1.0:1789760766.667831:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760766.667848:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5289, 5289] 00000008:00010000:1.0:1789760766.667968:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760766.667973:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760766.667984:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21663744, 21667840) append 0 append lockpos 0 00000080:00200000:1.0:1789760766.667997:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21663744=0x14a9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760766.668052:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760766.668056:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21667840 00000020:00010000:1.0:1789760766.668062:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760766.668066:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.668069:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c5417488/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760766.668072:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760766.668074:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760766.668081:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760766.668086:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760766.668112:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760766.668203:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 21663744<>21667840 00000100:00100000:1.0:1789760766.769978:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507311232:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760766.821145:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5291, 5291] 00000080:00200000:1.0:1789760766.821166:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760766.821173:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000100:00100000:1.0:1789760767.100563:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507312128:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760767.299205:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5301, 5301] 00000080:00200000:1.0:1789760767.299223:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760767.299230:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760767.299248:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21716992 i_blocks 43015 00000020:00200000:1.0:1789760767.299253:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760767.299323:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760767.299413:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21716992, count: 4096 00000080:00200000:1.0:1789760767.299422:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21716992, bytes: 4096 00000080:00200000:1.0:1789760767.299438:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760767.299447:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760767.299449:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21716992, 21721088) 0 0 00000080:00200000:1.0:1789760767.299453:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760767.299470:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5302, 5302] 00020000:00200000:1.0:1789760767.299479:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21716992 chunk: [21716992, 21721088) 21721088, 4096 00020000:00200000:1.0:1789760767.299484:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760767.299505:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5302, 5302] 00000008:00010000:1.0:1789760767.299598:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760767.299606:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760767.299620:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21716992, 21721088) append 0 append lockpos 0 00000080:00200000:1.0:1789760767.299636:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21716992=0x14b6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760767.299703:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760767.299706:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21721088 00000020:00010000:1.0:1789760767.299715:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760767.299718:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760767.299722:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760767.299726:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760767.299729:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760767.299737:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760767.299745:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760767.299772:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760767.301278:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 21716992<>21721088 00000080:00200000:1.0:1789760767.452694:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5304, 5304] 00000080:00200000:1.0:1789760767.452781:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760767.453026:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760767.453044:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21729280 i_blocks 43015 00000020:00200000:1.0:1789760767.453048:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760767.453069:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760767.453107:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21729280, count: 4096 00000080:00200000:1.0:1789760767.453114:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21729280, bytes: 4096 00000080:00200000:1.0:1789760767.453126:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760767.453136:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760767.453138:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21729280, 21733376) 0 0 00000080:00200000:1.0:1789760767.453142:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760767.453158:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5305, 5305] 00020000:00200000:1.0:1789760767.453294:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21729280 chunk: [21729280, 21733376) 21733376, 4096 00020000:00200000:1.0:1789760767.453301:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760767.453323:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5305, 5305] 00000008:00010000:1.0:1789760767.453343:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760767.453349:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000080:00200000:1.0:1789760767.453361:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21729280, 21733376) append 0 append lockpos 0 00000080:00200000:1.0:1789760767.453411:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21729280=0x14b9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760767.453472:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760767.453475:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21733376 00000020:00010000:1.0:1789760767.453483:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760767.453486:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760767.453489:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e610b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760767.453492:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760767.453495:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e610b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760767.453503:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760767.453510:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760767.453532:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760767.453645:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 21729280<>21733376 00000080:00200000:1.0:1789760767.500708:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5305, 5305] 00000080:00200000:1.0:1789760767.500736:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760767.500744:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760767.500761:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21733376 i_blocks 43015 00000020:00200000:1.0:1789760767.500767:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760767.500784:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760767.500831:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21733376, count: 4096 00000080:00200000:1.0:1789760767.500841:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21733376, bytes: 4096 00000080:00200000:1.0:1789760767.500856:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760767.500865:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760767.500867:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21733376, 21737472) 0 0 00000080:00200000:1.0:1789760767.500872:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760767.500889:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5306, 5306] 00020000:00200000:1.0:1789760767.500897:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21733376 chunk: [21733376, 21737472) 21737472, 4096 00020000:00200000:1.0:1789760767.500902:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760767.500922:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5306, 5306] 00000008:00010000:1.0:1789760767.500943:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760767.501179:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e610b0, osc lock ffff8d25c57ee4c0, flags 0x0 00000100:00100000:1.0:1789760767.800209:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972632 gen 3 00000008:00010000:1.0:1789760767.800257:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760767.800265:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760767.800278:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760767.800356:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760767.800427:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507314560:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760767.842078:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5313, 5313] 00000080:00200000:1.0:1789760767.842095:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760767.842102:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760767.842118:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21766144 i_blocks 43015 00000020:00200000:1.0:1789760767.842122:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760767.842132:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760767.842168:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21766144, count: 4096 00000080:00200000:1.0:1789760767.842175:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21766144, bytes: 4096 00000080:00200000:1.0:1789760767.842185:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760767.842194:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760767.842196:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21766144, 21770240) 0 0 00000080:00200000:1.0:1789760767.842201:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760767.842218:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5314, 5314] 00020000:00200000:1.0:1789760767.842227:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21766144 chunk: [21766144, 21770240) 21770240, 4096 00020000:00200000:1.0:1789760767.842232:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760767.842254:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5314, 5314] 00000008:00010000:1.0:1789760767.842272:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760767.842278:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760767.842291:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21766144, 21770240) append 0 append lockpos 0 00000080:00200000:1.0:1789760767.842306:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21766144=0x14c2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760767.842364:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760767.842367:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21770240 00000020:00010000:1.0:1789760767.842836:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760767.842842:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760767.842846:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760767.842850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760767.842853:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760767.842864:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760767.842873:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760767.842892:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760767.842993:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 21766144<>21770240 00000100:00100000:1.0:1789760767.843032:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760767.843052:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507314816:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760767.876474:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760767.876491:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760767.876505:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760767.876572:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760767.876606:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507314816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760767.877358:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507314944:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760767.931416:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972635 gen 3 00000008:00010000:1.0:1789760767.931658:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760767.931667:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760767.931678:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760767.931745:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760767.931777:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507314944:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760767.933292:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315072:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760767.973676:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972636 gen 3 00000008:00010000:1.0:1789760767.973924:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760767.973933:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760767.973946:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760767.973969:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760767.974003:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315072:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760767.975311:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760767.975338:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315200:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.012306:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972637 gen 3 00000008:00010000:1.0:1789760768.012359:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760768.012368:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.012411:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.012429:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760768.012461:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315200:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.014435:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315328:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760768.059233:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760768.059249:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.059268:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.059296:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760768.059334:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315328:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.060710:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507315456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.092813:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972638 gen 3 00000100:00100000:1.0:1789760768.093211:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b8000 x1876699507315456/t4294972639(4294972639) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760784 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760768.093268:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760768.093276:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.093289:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.093312:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760768.093341:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507315456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.096306:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.129647:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972640 gen 3 00000008:00010000:1.0:1789760768.129794:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760768.129804:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.129815:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.130112:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760768.130148:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507315584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.131656:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507315712:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760768.173935:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760768.173952:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.173965:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.173992:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760768.174022:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507315712:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760768.206277:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5322, 5322] 00000080:00200000:1.0:1789760768.206294:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760768.206300:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760768.206317:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21803008 i_blocks 43015 00000020:00200000:1.0:1789760768.206321:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760768.206331:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760768.206366:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21803008, count: 4096 00000080:00200000:1.0:1789760768.206373:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21803008, bytes: 4096 00000080:00200000:1.0:1789760768.206417:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760768.206427:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760768.206429:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21803008, 21807104) 0 0 00000080:00200000:1.0:1789760768.206434:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760768.206451:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5323, 5323] 00020000:00200000:1.0:1789760768.206460:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21803008 chunk: [21803008, 21807104) 21807104, 4096 00020000:00200000:1.0:1789760768.206465:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760768.206486:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5323, 5323] 00000008:00010000:1.0:1789760768.206505:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760768.206511:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760768.206523:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21803008, 21807104) append 0 append lockpos 0 00000080:00200000:1.0:1789760768.206537:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21803008=0x14cb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760768.206594:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760768.206597:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21807104 00000020:00010000:1.0:1789760768.206605:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760768.206609:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.206616:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.206620:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760768.206624:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760768.206633:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760768.206641:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760768.206658:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760768.206852:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 21803008<>21807104 00000100:00100000:1.0:1789760768.206898:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760768.206917:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507315968:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.243294:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972643 gen 3 00000008:00010000:1.0:1789760768.243348:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760768.243356:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.243367:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.243426:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760768.243455:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507315968:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760768.495924:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5328, 5328] 00000080:00200000:1.0:1789760768.495940:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760768.495947:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760768.495962:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21827584 i_blocks 43015 00000020:00200000:1.0:1789760768.495967:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760768.495976:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760768.496012:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21827584, count: 4096 00000080:00200000:1.0:1789760768.496020:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21827584, bytes: 4096 00000080:00200000:1.0:1789760768.496031:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760768.496039:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760768.496042:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21827584, 21831680) 0 0 00000080:00200000:1.0:1789760768.496047:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760768.496064:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5329, 5329] 00020000:00200000:1.0:1789760768.496073:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21827584 chunk: [21827584, 21831680) 21831680, 4096 00020000:00200000:1.0:1789760768.496077:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760768.496097:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5329, 5329] 00000008:00010000:1.0:1789760768.496116:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760768.496123:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760768.496135:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21827584, 21831680) append 0 append lockpos 0 00000080:00200000:1.0:1789760768.496150:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21827584=0x14d1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760768.496206:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760768.496209:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21831680 00000020:00010000:1.0:1789760768.496218:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760768.496221:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.496224:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.496228:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760768.496231:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760768.496239:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760768.496246:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760768.496263:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760768.496369:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 21827584<>21831680 00000100:00100000:1.0:1789760768.568041:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972649 gen 3 00000008:00010000:1.0:1789760768.568094:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760768.568103:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.568114:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.568143:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760768.568176:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507316736:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760768.759265:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5333, 5333] 00000080:00200000:1.0:1789760768.759282:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760768.759289:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760768.759305:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21848064 i_blocks 43015 00000020:00200000:1.0:1789760768.759309:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760768.759319:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760768.759352:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21848064, count: 4096 00000080:00200000:1.0:1789760768.759359:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21848064, bytes: 4096 00000080:00200000:1.0:1789760768.759369:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760768.759410:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760768.759412:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21848064, 21852160) 0 0 00000080:00200000:1.0:1789760768.759417:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760768.759434:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5334, 5334] 00020000:00200000:1.0:1789760768.759442:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21848064 chunk: [21848064, 21852160) 21852160, 4096 00020000:00200000:1.0:1789760768.759447:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760768.759467:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5334, 5334] 00000008:00010000:1.0:1789760768.759485:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760768.759490:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760768.759502:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21848064, 21852160) append 0 append lockpos 0 00000080:00200000:1.0:1789760768.759517:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21848064=0x14d6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760768.759574:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760768.759577:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21852160 00000020:00010000:1.0:1789760768.759586:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760768.759590:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.759593:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.759597:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760768.759600:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760768.759608:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760768.759616:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760768.759634:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760768.759741:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 21848064<>21852160 00000008:00010000:1.0:1789760768.841675:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760768.841691:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.841702:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.841727:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760768.841757:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507317376:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760768.905693:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5335, 5335] 00000080:00200000:1.0:1789760768.905710:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760768.905716:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760768.905732:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21856256 i_blocks 43015 00000020:00200000:1.0:1789760768.905736:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760768.905747:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760768.905791:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21856256, count: 4096 00000080:00200000:1.0:1789760768.905798:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21856256, bytes: 4096 00000080:00200000:1.0:1789760768.905808:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760768.905818:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760768.905820:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21856256, 21860352) 0 0 00000080:00200000:1.0:1789760768.905825:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760768.905841:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5336, 5336] 00020000:00200000:1.0:1789760768.905851:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21856256 chunk: [21856256, 21860352) 21860352, 4096 00020000:00200000:1.0:1789760768.905856:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760768.905875:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5336, 5336] 00000008:00010000:1.0:1789760768.905894:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760768.905900:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760768.905912:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21856256, 21860352) append 0 append lockpos 0 00000080:00200000:1.0:1789760768.905926:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21856256=0x14d8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760768.905985:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760768.905988:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21860352 00000020:00010000:1.0:1789760768.905997:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760768.906002:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.906005:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760768.906009:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760768.906012:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760768.906020:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760768.906027:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760768.906044:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760768.906165:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 21856256<>21860352 00000100:00100000:1.0:1789760768.906211:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760768.906228:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507317632:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760768.958371:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972656 gen 3 00000008:00010000:1.0:1789760768.958490:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760768.958496:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760768.958505:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760768.958822:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760768.958848:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507317632:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760769.145357:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5340, 5340] 00000080:00200000:1.0:1789760769.145372:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760769.145412:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760769.145425:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21876736 i_blocks 43015 00000020:00200000:1.0:1789760769.145429:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760769.145439:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760769.145464:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21876736, count: 4096 00000080:00200000:1.0:1789760769.145470:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21876736, bytes: 4096 00000080:00200000:1.0:1789760769.145477:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760769.145484:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760769.145486:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21876736, 21880832) 0 0 00000080:00200000:1.0:1789760769.145490:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760769.145503:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5341, 5341] 00020000:00200000:1.0:1789760769.145510:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21876736 chunk: [21876736, 21880832) 21880832, 4096 00020000:00200000:1.0:1789760769.145514:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760769.145529:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5341, 5341] 00000008:00010000:1.0:1789760769.145543:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760769.145548:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760769.145557:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21876736, 21880832) append 0 append lockpos 0 00000080:00200000:1.0:1789760769.145568:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21876736=0x14dd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760769.145618:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760769.145621:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21880832 00000020:00010000:1.0:1789760769.145627:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760769.145630:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760769.145633:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760769.145637:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760769.145639:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760769.145646:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760769.145651:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760769.145666:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760769.145739:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 21876736<>21880832 00000100:00100000:1.0:1789760769.220368:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972661 gen 3 00000008:00010000:1.0:1789760769.220440:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760769.220448:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760769.220457:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760769.220484:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760769.220508:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507318272:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760769.260175:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5342, 5342] 00000080:00200000:1.0:1789760769.260188:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760769.260193:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760769.260206:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21884928 i_blocks 43015 00000020:00200000:1.0:1789760769.260210:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760769.260218:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760769.260313:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21884928, count: 4096 00000080:00200000:1.0:1789760769.260321:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21884928, bytes: 4096 00000080:00200000:1.0:1789760769.260329:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760769.260336:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760769.260337:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21884928, 21889024) 0 0 00000080:00200000:1.0:1789760769.260341:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760769.260354:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5343, 5343] 00020000:00200000:1.0:1789760769.260361:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21884928 chunk: [21884928, 21889024) 21889024, 4096 00020000:00200000:1.0:1789760769.260364:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760769.260399:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5343, 5343] 00000008:00010000:1.0:1789760769.260415:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760769.260419:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760769.260429:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21884928, 21889024) append 0 append lockpos 0 00000080:00200000:1.0:1789760769.260441:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21884928=0x14df000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760769.260484:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760769.260487:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21889024 00000020:00010000:1.0:1789760769.260495:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760769.260498:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760769.260501:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760769.260504:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760769.260506:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760769.260513:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760769.260518:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760769.260532:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760769.260612:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 21884928<>21889024 00000008:00010000:1.0:1789760769.334636:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760769.334645:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760769.334652:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760769.334718:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760769.334742:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507318528:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760769.665154:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5348, 5348] 00000080:00200000:1.0:1789760769.665168:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760769.665174:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760769.665186:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21909504 i_blocks 43015 00000020:00200000:1.0:1789760769.665191:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760769.665200:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760769.665250:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21909504, count: 4096 00000080:00200000:1.0:1789760769.665257:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21909504, bytes: 4096 00000080:00200000:1.0:1789760769.665266:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760769.665275:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760769.665277:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21909504, 21913600) 0 0 00000080:00200000:1.0:1789760769.665282:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760769.665296:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5349, 5349] 00020000:00200000:1.0:1789760769.665303:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21909504 chunk: [21909504, 21913600) 21913600, 4096 00020000:00200000:1.0:1789760769.665306:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760769.665322:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5349, 5349] 00000008:00010000:1.0:1789760769.665335:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760769.665341:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760769.665349:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21909504, 21913600) append 0 append lockpos 0 00000080:00200000:1.0:1789760769.665360:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21909504=0x14e5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760769.665426:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760769.665429:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21913600 00000020:00010000:1.0:1789760769.665436:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760769.665439:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760769.665441:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760769.665445:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760769.665447:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760769.665454:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760769.665459:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760769.665473:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760769.665552:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 21909504<>21913600 00000100:00100000:1.0:1789760769.665597:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760769.665615:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507319296:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760769.734081:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760769.734093:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760769.734104:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760769.734132:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760769.734159:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507319296:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.050128:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972675 gen 3 00000008:00010000:1.0:1789760770.050168:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760770.050176:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.050187:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.050279:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760770.050309:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320064:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.061551:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.103802:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972676 gen 3 00000008:00010000:1.0:1789760770.103859:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760770.103869:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.103882:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.103911:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760770.103941:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.104924:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507320320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.138837:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972677 gen 3 00000008:00010000:1.0:1789760770.138892:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760770.138913:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.138926:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.138948:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760770.138980:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507320320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.140491:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320448:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.174447:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972678 gen 3 00000008:00010000:1.0:1789760770.174499:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760770.174508:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.174521:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.174545:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760770.174574:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320448:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.175449:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507320576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.208749:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972679 gen 3 00000008:00010000:1.0:1789760770.208869:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760770.208877:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.208890:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.208949:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760770.208979:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507320576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.209718:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320704:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.238510:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972680 gen 3 00000008:00010000:1.0:1789760770.238553:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760770.238560:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.238572:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.238593:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760770.238619:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320704:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.239491:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507320832:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760770.269594:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760770.269611:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.269624:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.269649:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760770.269679:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507320832:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.270289:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320960:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.317935:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972682 gen 3 00000008:00010000:1.0:1789760770.318000:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760770.318007:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.318019:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.318045:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760770.318074:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507320960:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.318934:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507321088:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.381472:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972683 gen 3 00000008:00010000:1.0:1789760770.381526:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760770.381535:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.381548:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.381631:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760770.381663:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507321088:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.385816:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507321216:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760770.427842:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760770.427861:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.427873:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.427894:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760770.427927:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507321216:192.168.203.141@tcp:4:dd.0 00000080:00200008:1.0:1789760770.428161:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21975040, count: 4096 00000080:00200000:1.0:1789760770.428170:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21975040, bytes: 4096 00000080:00200000:1.0:1789760770.428182:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760770.428190:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760770.428193:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21975040, 21979136) 0 0 00000080:00200000:1.0:1789760770.428198:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760770.428228:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5365, 5365] 00020000:00200000:1.0:1789760770.428236:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21975040 chunk: [21975040, 21979136) 21979136, 4096 00020000:00200000:1.0:1789760770.428241:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760770.428263:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5365, 5365] 00000008:00010000:1.0:1789760770.428282:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760770.428287:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760770.428298:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21975040, 21979136) append 0 append lockpos 0 00000080:00200000:1.0:1789760770.428312:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21975040=0x14f5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760770.428366:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760770.428369:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 21979136 00000020:00010000:1.0:1789760770.428408:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760770.428412:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760770.428415:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760770.428418:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760770.428420:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760770.428428:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760770.428435:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760770.428447:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760770.428552:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 21975040<>21979136 00000100:00100000:1.0:1789760770.428579:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760770.428595:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507321344:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.491657:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972685 gen 3 00000008:00010000:1.0:1789760770.491719:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760770.491728:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.491737:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.491762:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760770.491801:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507321344:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.585118:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972687 gen 3 00000008:00010000:1.0:1789760770.585181:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760770.585189:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.585200:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.585227:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760770.585259:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507321600:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760770.777220:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5370, 5370] 00000080:00200000:1.0:1789760770.777237:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760770.777244:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760770.777261:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 21999616 i_blocks 43015 00000020:00200000:1.0:1789760770.777266:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760770.777277:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760770.777326:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 21999616, count: 4096 00000080:00200000:1.0:1789760770.777335:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 21999616, bytes: 4096 00000080:00200000:1.0:1789760770.777356:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760770.777366:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760770.777369:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [21999616, 22003712) 0 0 00000080:00200000:1.0:1789760770.777374:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760770.777434:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5371, 5371] 00020000:00200000:1.0:1789760770.777446:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 21999616 chunk: [21999616, 22003712) 22003712, 4096 00020000:00200000:1.0:1789760770.777452:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760770.777475:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5371, 5371] 00000008:00010000:1.0:1789760770.777499:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760770.777506:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760770.777519:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [21999616, 22003712) append 0 append lockpos 0 00000080:00200000:1.0:1789760770.777535:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=21999616=0x14fb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760770.777607:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760770.777610:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22003712 00000020:00010000:1.0:1789760770.777618:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760770.777622:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760770.777625:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760770.777629:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760770.777632:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760770.777641:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760770.777649:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760770.777669:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760770.777803:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 21999616<>22003712 00000100:00100000:1.0:1789760770.777849:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760770.777868:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507322112:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.820725:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972691 gen 3 00000008:00010000:1.0:1789760770.820794:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760770.820810:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760770.820822:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760770.820850:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760770.820882:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507322112:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760770.952156:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507322496:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760771.003791:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5374, 5374] 00000080:00200000:1.0:1789760771.003808:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.003815:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.003831:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22016000 i_blocks 43015 00000020:00200000:1.0:1789760771.003835:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.003845:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.003879:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22016000, count: 4096 00000080:00200000:1.0:1789760771.003887:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22016000, bytes: 4096 00000080:00200000:1.0:1789760771.004132:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.004142:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.004144:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22016000, 22020096) 0 0 00000080:00200000:1.0:1789760771.004149:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.004167:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5375, 5375] 00020000:00200000:1.0:1789760771.004175:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22016000 chunk: [22016000, 22020096) 22020096, 4096 00020000:00200000:1.0:1789760771.004179:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.004200:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5375, 5375] 00000008:00010000:1.0:1789760771.004220:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.004226:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.004239:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22016000, 22020096) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.004256:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22016000=0x14ff000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.004319:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.004322:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22020096 00000020:00010000:1.0:1789760771.004330:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.004334:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.004511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.004516:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.004519:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.004529:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.004536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.004559:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760771.004671:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 22016000<>22020096 00000080:00200000:1.0:1789760771.051061:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5375, 5375] 00000080:00200000:1.0:1789760771.051077:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.051082:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.051097:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22020096 i_blocks 43015 00000020:00200000:1.0:1789760771.051100:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.051109:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.051143:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22020096, count: 4096 00000080:00200000:1.0:1789760771.051149:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22020096, bytes: 4096 00000080:00200000:1.0:1789760771.051161:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.051168:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.051170:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22020096, 22024192) 0 0 00000080:00200000:1.0:1789760771.051174:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.051188:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5376, 5376] 00020000:00200000:1.0:1789760771.051248:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22020096 chunk: [22020096, 22024192) 22024192, 4096 00020000:00200000:1.0:1789760771.051252:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.051272:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5376, 5376] 00000008:00010000:1.0:1789760771.051289:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.051295:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.051308:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22020096, 22024192) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.051323:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22020096=0x1500000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.051476:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.051480:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22024192 00000020:00010000:1.0:1789760771.051619:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.051624:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.051628:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.051632:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.051635:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.051645:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.051652:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.051676:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760771.051802:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 22020096<>22024192 00000080:00200000:1.0:1789760771.136112:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5377, 5377] 00000080:00200000:1.0:1789760771.136129:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.136135:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.136152:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22028288 i_blocks 45063 00000020:00200000:1.0:1789760771.136156:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.136166:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.136201:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22028288, count: 4096 00000080:00200000:1.0:1789760771.136210:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22028288, bytes: 4096 00000080:00200000:1.0:1789760771.136221:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.136231:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.136233:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22028288, 22032384) 0 0 00000080:00200000:1.0:1789760771.136238:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.136255:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5378, 5378] 00020000:00200000:1.0:1789760771.136265:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22028288 chunk: [22028288, 22032384) 22032384, 4096 00020000:00200000:1.0:1789760771.136270:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.136290:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5378, 5378] 00000008:00010000:1.0:1789760771.136309:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.136315:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.136326:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22028288, 22032384) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.136341:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22028288=0x1502000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.136424:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.136428:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22032384 00000020:00010000:1.0:1789760771.136437:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.136441:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.136444:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.136448:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.136451:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.136460:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.136467:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.136484:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760771.136658:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 22028288<>22032384 00000080:00200000:1.0:1789760771.193412:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5378, 5378] 00000080:00200000:1.0:1789760771.193430:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.193436:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.193452:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22032384 i_blocks 45063 00000020:00200000:1.0:1789760771.193457:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.193468:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.193503:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22032384, count: 4096 00000080:00200000:1.0:1789760771.193512:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22032384, bytes: 4096 00000080:00200000:1.0:1789760771.193526:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.193537:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.193539:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22032384, 22036480) 0 0 00000080:00200000:1.0:1789760771.193544:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.193560:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5379, 5379] 00020000:00200000:1.0:1789760771.193569:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22032384 chunk: [22032384, 22036480) 22036480, 4096 00020000:00200000:1.0:1789760771.193574:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.193668:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5379, 5379] 00000008:00010000:1.0:1789760771.193690:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.193696:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.193709:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22032384, 22036480) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.193788:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22032384=0x1503000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.193924:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.193928:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22036480 00000020:00010000:1.0:1789760771.193937:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.193940:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.193943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.193947:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.193949:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.193958:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.193965:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.193987:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760771.194208:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 22032384<>22036480 00000080:00200000:1.0:1789760771.229065:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5379, 5379] 00000080:00200000:1.0:1789760771.229083:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.229090:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.229107:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22036480 i_blocks 45063 00000020:00200000:1.0:1789760771.229111:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.229121:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.229155:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22036480, count: 4096 00000080:00200000:1.0:1789760771.229163:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22036480, bytes: 4096 00000080:00200000:1.0:1789760771.229174:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.229184:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.229186:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22036480, 22040576) 0 0 00000080:00200000:1.0:1789760771.229192:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.229270:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5380, 5380] 00020000:00200000:1.0:1789760771.229280:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22036480 chunk: [22036480, 22040576) 22040576, 4096 00020000:00200000:1.0:1789760771.229284:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.229307:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5380, 5380] 00000008:00010000:1.0:1789760771.229325:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.229332:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.229342:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22036480, 22040576) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.229358:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22036480=0x1504000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.229450:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.229454:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22040576 00000020:00010000:1.0:1789760771.229464:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.229467:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.229470:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.229474:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.229477:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.229485:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.229493:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.229510:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760771.229626:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 22036480<>22040576 00000080:00200000:1.0:1789760771.259624:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5380, 5380] 00000080:00200000:1.0:1789760771.259642:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.259648:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.259664:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22040576 i_blocks 45063 00000020:00200000:1.0:1789760771.259668:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.259677:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.259713:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22040576, count: 4096 00000080:00200000:1.0:1789760771.259720:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22040576, bytes: 4096 00000080:00200000:1.0:1789760771.259728:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.259735:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.259737:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22040576, 22044672) 0 0 00000080:00200000:1.0:1789760771.259741:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.259759:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5381, 5381] 00020000:00200000:1.0:1789760771.259780:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22040576 chunk: [22040576, 22044672) 22044672, 4096 00020000:00200000:1.0:1789760771.259785:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.259805:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5381, 5381] 00000008:00010000:1.0:1789760771.259821:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.259827:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.259837:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22040576, 22044672) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.259852:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22040576=0x1505000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.259981:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.259984:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22044672 00000020:00010000:1.0:1789760771.259993:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.259996:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.259999:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.260003:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.260006:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.260014:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.260021:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.260038:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760771.260297:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 22040576<>22044672 00000080:00200000:1.0:1789760771.310560:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5381, 5381] 00000080:00200000:1.0:1789760771.310586:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.310593:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.310610:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22044672 i_blocks 45063 00000020:00200000:1.0:1789760771.310615:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.310630:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.310664:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22044672, count: 4096 00000080:00200000:1.0:1789760771.310674:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22044672, bytes: 4096 00000080:00200000:1.0:1789760771.310689:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.310699:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.310701:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22044672, 22048768) 0 0 00000080:00200000:1.0:1789760771.310706:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.311517:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5382, 5382] 00020000:00200000:1.0:1789760771.311529:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22044672 chunk: [22044672, 22048768) 22048768, 4096 00020000:00200000:1.0:1789760771.311533:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.311559:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5382, 5382] 00000008:00010000:1.0:1789760771.311579:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.311585:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.311596:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22044672, 22048768) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.311611:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22044672=0x1506000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.311688:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.311692:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22048768 00000020:00010000:1.0:1789760771.311700:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.311704:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.311707:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.311948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.311985:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.311998:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.312006:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.312108:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760771.312221:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 22044672<>22048768 00000080:00200000:1.0:1789760771.358669:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5382, 5382] 00000080:00200000:1.0:1789760771.358687:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.358693:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.358710:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22048768 i_blocks 45063 00000020:00200000:1.0:1789760771.358714:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.358726:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.358774:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22048768, count: 4096 00000080:00200000:1.0:1789760771.358782:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22048768, bytes: 4096 00000080:00200000:1.0:1789760771.358792:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.358802:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.358804:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22048768, 22052864) 0 0 00000080:00200000:1.0:1789760771.358809:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.358825:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5383, 5383] 00020000:00200000:1.0:1789760771.358834:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22048768 chunk: [22048768, 22052864) 22052864, 4096 00020000:00200000:1.0:1789760771.358839:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.358925:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5383, 5383] 00000008:00010000:1.0:1789760771.358944:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.359021:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.359036:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22048768, 22052864) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.359053:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22048768=0x1507000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.359116:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.359119:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22052864 00000020:00010000:1.0:1789760771.359128:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.359131:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.359134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.359137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.359140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.359148:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.359155:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.359174:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760771.359286:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 22048768<>22052864 00000008:00010000:2.0:1789760771.414452:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760771.414459:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760771.414472:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22052864, 22056960) append 0 append lockpos 0 00000080:00200000:2.0:1789760771.414487:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22052864=0x1508000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760771.414550:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760771.414553:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22056960 00000020:00010000:2.0:1789760771.414562:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760771.414565:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.414569:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.414572:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760771.414575:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760771.414583:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760771.414590:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760771.414607:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760771.414735:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 22052864<>22056960 00000080:00200000:2.0:1789760771.471867:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5384, 5384] 00000080:00200000:2.0:1789760771.471886:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760771.471892:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760771.471909:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22056960 i_blocks 45063 00000020:00200000:2.0:1789760771.471913:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760771.471923:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760771.471958:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22056960, count: 4096 00000080:00200000:2.0:1789760771.471965:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22056960, bytes: 4096 00000080:00200000:2.0:1789760771.471979:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760771.471987:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760771.471990:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22056960, 22061056) 0 0 00000080:00200000:2.0:1789760771.471994:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760771.472011:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5385, 5385] 00020000:00200000:2.0:1789760771.472019:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22056960 chunk: [22056960, 22061056) 22061056, 4096 00020000:00200000:2.0:1789760771.472023:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760771.472042:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5385, 5385] 00000008:00010000:2.0:1789760771.472965:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760771.472976:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760771.472993:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22056960, 22061056) append 0 append lockpos 0 00000080:00200000:2.0:1789760771.473012:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22056960=0x1509000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760771.473310:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760771.473313:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22061056 00000020:00010000:2.0:1789760771.473322:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760771.473326:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.473329:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.473332:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760771.473335:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760771.473345:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760771.473354:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760771.473472:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760771.473585:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 22056960<>22061056 00000080:00200000:2.0:1789760771.519152:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5385, 5385] 00000080:00200000:2.0:1789760771.519169:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760771.519175:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760771.519191:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22061056 i_blocks 45063 00000020:00200000:2.0:1789760771.519195:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760771.519211:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760771.519247:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22061056, count: 4096 00000080:00200000:2.0:1789760771.519254:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22061056, bytes: 4096 00000080:00200000:2.0:1789760771.519263:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760771.519274:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760771.519276:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22061056, 22065152) 0 0 00000080:00200000:2.0:1789760771.519281:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760771.519298:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5386, 5386] 00020000:00200000:2.0:1789760771.519306:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22061056 chunk: [22061056, 22065152) 22065152, 4096 00020000:00200000:2.0:1789760771.519310:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760771.519332:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5386, 5386] 00000008:00010000:2.0:1789760771.519350:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760771.519356:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760771.519368:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22061056, 22065152) append 0 append lockpos 0 00000080:00200000:2.0:1789760771.519465:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22061056=0x150a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760771.519596:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760771.519599:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22065152 00000020:00010000:2.0:1789760771.519608:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760771.519611:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.519614:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.519617:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760771.519620:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760771.519628:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760771.519636:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760771.519653:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760771.519757:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 22061056<>22065152 00000080:00200000:2.0:1789760771.582343:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5386, 5386] 00000080:00200000:2.0:1789760771.582361:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760771.582366:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760771.582408:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22065152 i_blocks 45063 00000020:00200000:2.0:1789760771.582413:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760771.582423:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760771.582455:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22065152, count: 4096 00000080:00200000:2.0:1789760771.582462:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22065152, bytes: 4096 00000080:00200000:2.0:1789760771.582472:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760771.582480:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760771.582483:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22065152, 22069248) 0 0 00000080:00200000:2.0:1789760771.582487:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760771.582503:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5387, 5387] 00020000:00200000:2.0:1789760771.582512:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22065152 chunk: [22065152, 22069248) 22069248, 4096 00020000:00200000:2.0:1789760771.582516:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760771.582537:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5387, 5387] 00000008:00010000:2.0:1789760771.582556:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760771.582563:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760771.582574:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22065152, 22069248) append 0 append lockpos 0 00000080:00200000:2.0:1789760771.582588:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22065152=0x150b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760771.583008:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760771.583013:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22069248 00000020:00010000:2.0:1789760771.583023:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760771.583027:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.583030:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.583034:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760771.583037:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760771.583046:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760771.583054:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760771.583076:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760771.585168:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 22065152<>22069248 00000080:00200000:2.0:1789760771.651142:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5387, 5387] 00000080:00200000:2.0:1789760771.651160:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760771.651166:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760771.651183:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22069248 i_blocks 45063 00000020:00200000:2.0:1789760771.651188:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760771.651198:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760771.651246:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22069248, count: 4096 00000080:00200000:2.0:1789760771.651254:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22069248, bytes: 4096 00000080:00200000:2.0:1789760771.651266:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760771.651274:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760771.651276:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22069248, 22073344) 0 0 00000080:00200000:2.0:1789760771.651281:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760771.651298:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5388, 5388] 00020000:00200000:2.0:1789760771.651308:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22069248 chunk: [22069248, 22073344) 22073344, 4096 00020000:00200000:2.0:1789760771.651312:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760771.651338:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5388, 5388] 00000008:00010000:2.0:1789760771.651357:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760771.651364:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760771.651416:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22069248, 22073344) append 0 append lockpos 0 00000080:00200000:2.0:1789760771.651433:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22069248=0x150c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760771.651493:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760771.651496:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22073344 00000020:00010000:2.0:1789760771.651505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760771.651508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.651511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760771.651515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d015e998 (ffff8d25c5417488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760771.651519:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c5417488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760771.651527:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760771.651534:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760771.651553:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760771.651694:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 22069248<>22073344 00000100:00100000:2.0:1789760771.651739:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760771.651768:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507324288:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760771.682567:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac900 00000020:00200000:2.0:1789760771.682582:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760771.682592:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760771.682614:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac900 00000100:00100000:2.0:1789760771.682643:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507324288:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760771.733726:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000020:00200000:2.0:1789760771.733735:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760771.733746:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760771.733954:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000100:00100000:2.0:1789760771.733983:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507324416:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760771.735253:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507324544:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760771.804148:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972710 gen 3 00000008:00010000:2.0:1789760771.804223:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c900 00000020:00200000:2.0:1789760771.804231:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760771.804243:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760771.804893:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c900 00000100:00100000:2.0:1789760771.804934:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507324544:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760771.864313:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5391, 5391] 00000080:00200000:1.0:1789760771.864343:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.864351:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.864369:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22085632 i_blocks 45063 00000020:00200000:1.0:1789760771.864374:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.864407:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.864443:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22085632, count: 4096 00000080:00200000:1.0:1789760771.864453:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22085632, bytes: 4096 00000080:00200000:1.0:1789760771.864472:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.864482:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.864484:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22085632, 22089728) 0 0 00000080:00200000:1.0:1789760771.864489:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.864507:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5392, 5392] 00020000:00200000:1.0:1789760771.864516:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22085632 chunk: [22085632, 22089728) 22089728, 4096 00020000:00200000:1.0:1789760771.864522:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.864544:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5392, 5392] 00000008:00010000:1.0:1789760771.864565:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.864572:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.864585:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22085632, 22089728) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.864601:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22085632=0x1510000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.864677:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.864806:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22089728 00000020:00010000:1.0:1789760771.864818:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.864822:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.864825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.864829:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.864832:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.864842:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.864850:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.864884:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760771.865003:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22085632<>22089728 00000100:00100000:1.0:1789760771.865066:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760771.865095:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507324800:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760771.930486:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5392, 5392] 00000080:00200000:1.0:1789760771.930501:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.930507:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.930522:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22089728 i_blocks 45063 00000020:00200000:1.0:1789760771.930527:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.930537:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.930574:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22089728, count: 4096 00000080:00200000:1.0:1789760771.930581:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22089728, bytes: 4096 00000080:00200000:1.0:1789760771.930592:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.930601:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.930603:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22089728, 22093824) 0 0 00000080:00200000:1.0:1789760771.930608:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.930626:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5393, 5393] 00020000:00200000:1.0:1789760771.930636:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22089728 chunk: [22089728, 22093824) 22093824, 4096 00020000:00200000:1.0:1789760771.930640:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.930661:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5393, 5393] 00000008:00010000:1.0:1789760771.930681:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.930687:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.930699:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22089728, 22093824) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.930714:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22089728=0x1511000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.930785:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.930788:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22093824 00000020:00010000:1.0:1789760771.930797:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.930800:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.930803:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.930807:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.930810:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.930818:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.930825:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.930842:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760771.930952:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 22089728<>22093824 00000080:00200000:1.0:1789760771.964492:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5393, 5393] 00000080:00200000:1.0:1789760771.964509:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760771.964516:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760771.964533:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22093824 i_blocks 45063 00000020:00200000:1.0:1789760771.964538:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760771.964549:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760771.964582:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22093824, count: 4096 00000080:00200000:1.0:1789760771.964590:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22093824, bytes: 4096 00000080:00200000:1.0:1789760771.964600:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760771.964609:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760771.964611:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22093824, 22097920) 0 0 00000080:00200000:1.0:1789760771.964616:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760771.964638:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5394, 5394] 00020000:00200000:1.0:1789760771.964648:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22093824 chunk: [22093824, 22097920) 22097920, 4096 00020000:00200000:1.0:1789760771.964653:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760771.964673:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5394, 5394] 00000008:00010000:1.0:1789760771.964693:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760771.964700:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760771.964712:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22093824, 22097920) append 0 append lockpos 0 00000080:00200000:1.0:1789760771.964995:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22093824=0x1512000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760771.965068:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760771.965072:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22097920 00000020:00010000:1.0:1789760771.965081:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760771.965085:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.965088:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760771.965092:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760771.965095:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760771.965103:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760771.965110:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760771.965128:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760771.965236:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 22093824<>22097920 00000080:00200000:1.0:1789760772.012836:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5394, 5394] 00000080:00200000:1.0:1789760772.012853:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760772.012859:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760772.012877:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22097920 i_blocks 45063 00000020:00200000:1.0:1789760772.012881:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760772.012893:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760772.012930:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22097920, count: 4096 00000080:00200000:1.0:1789760772.012938:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22097920, bytes: 4096 00000080:00200000:1.0:1789760772.012949:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760772.012957:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760772.012959:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22097920, 22102016) 0 0 00000080:00200000:1.0:1789760772.012964:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760772.012982:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5395, 5395] 00020000:00200000:1.0:1789760772.012992:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22097920 chunk: [22097920, 22102016) 22102016, 4096 00020000:00200000:1.0:1789760772.012997:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760772.013020:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5395, 5395] 00000008:00010000:1.0:1789760772.013041:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760772.013047:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760772.013060:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22097920, 22102016) append 0 append lockpos 0 00000080:00200000:1.0:1789760772.013078:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22097920=0x1513000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760772.013142:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760772.013145:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22102016 00000020:00010000:1.0:1789760772.013153:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760772.013157:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.013160:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.013164:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760772.013167:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760772.013176:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760772.013183:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760772.013199:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760772.013308:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 22097920<>22102016 00000080:00200000:1.0:1789760772.056272:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5395, 5395] 00000080:00200000:1.0:1789760772.056288:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760772.056294:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760772.056310:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22102016 i_blocks 45063 00000020:00200000:1.0:1789760772.056314:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760772.056324:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760772.056357:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22102016, count: 4096 00000080:00200000:1.0:1789760772.056365:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22102016, bytes: 4096 00000080:00200000:1.0:1789760772.056402:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760772.056413:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760772.056415:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22102016, 22106112) 0 0 00000080:00200000:1.0:1789760772.056420:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760772.056437:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5396, 5396] 00020000:00200000:1.0:1789760772.056446:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22102016 chunk: [22102016, 22106112) 22106112, 4096 00020000:00200000:1.0:1789760772.056450:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760772.056471:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5396, 5396] 00000008:00010000:1.0:1789760772.056491:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760772.056497:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760772.056509:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22102016, 22106112) append 0 append lockpos 0 00000080:00200000:1.0:1789760772.056524:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22102016=0x1514000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760772.056583:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760772.056586:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22106112 00000020:00010000:1.0:1789760772.056594:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760772.056597:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.056600:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.056604:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760772.056606:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760772.056614:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760772.056622:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760772.056640:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760772.056828:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 22102016<>22106112 00000080:00200000:1.0:1789760772.120011:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5396, 5396] 00000080:00200000:1.0:1789760772.120042:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760772.120049:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760772.120067:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22106112 i_blocks 45063 00000020:00200000:1.0:1789760772.120073:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760772.120089:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760772.120126:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22106112, count: 4096 00000080:00200000:1.0:1789760772.120135:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22106112, bytes: 4096 00000080:00200000:1.0:1789760772.120150:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760772.120160:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760772.120162:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22106112, 22110208) 0 0 00000080:00200000:1.0:1789760772.120167:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760772.120184:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5397, 5397] 00020000:00200000:1.0:1789760772.120193:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22106112 chunk: [22106112, 22110208) 22110208, 4096 00020000:00200000:1.0:1789760772.120206:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760772.120228:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5397, 5397] 00000008:00010000:1.0:1789760772.120248:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760772.120254:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760772.120266:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22106112, 22110208) append 0 append lockpos 0 00000080:00200000:1.0:1789760772.120280:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22106112=0x1515000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760772.120351:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760772.120354:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22110208 00000020:00010000:1.0:1789760772.120362:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_release():245 00000020:00010000:1.0:1789760772.120366:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.120370:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038cf8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.120374:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9c68 (ffff8d25c8038a28/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760772.120427:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760772.120438:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760772.120446:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760772.120469:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760772.120588:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 22106112<>22110208 00000080:00200000:1.0:1789760772.179551:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5397, 5397] 00000080:00200000:1.0:1789760772.179570:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760772.179576:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760772.179592:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22110208 i_blocks 45063 00000020:00200000:1.0:1789760772.179596:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760772.179607:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200000:2.0:1789760772.223939:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5398, 5398] 00000080:00200000:2.0:1789760772.223956:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.223963:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.223979:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22114304 i_blocks 45063 00000020:00200000:2.0:1789760772.223984:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.223993:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.224129:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22114304, count: 4096 00000080:00200000:2.0:1789760772.224138:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22114304, bytes: 4096 00000080:00200000:2.0:1789760772.224148:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.224157:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.224159:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22114304, 22118400) 0 0 00000080:00200000:2.0:1789760772.224163:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.224181:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5399, 5399] 00020000:00200000:2.0:1789760772.224191:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22114304 chunk: [22114304, 22118400) 22118400, 4096 00020000:00200000:2.0:1789760772.224195:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.224225:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5399, 5399] 00000008:00010000:2.0:1789760772.224243:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.224250:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.224261:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22114304, 22118400) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.224275:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22114304=0x1517000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.224332:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.224335:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22118400 00000020:00010000:2.0:1789760772.224343:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.224347:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.224352:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.224355:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.224358:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.224366:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.224372:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.224435:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760772.224553:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 22114304<>22118400 00000080:00200000:1.0:1789760772.268111:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5399, 5399] 00000080:00200000:1.0:1789760772.268131:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760772.268137:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760772.268154:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22118400 i_blocks 45063 00000020:00200000:1.0:1789760772.268158:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760772.268170:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760772.268277:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22118400, count: 4096 00000080:00200000:1.0:1789760772.268286:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22118400, bytes: 4096 00000080:00200000:1.0:1789760772.268300:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760772.268310:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760772.268313:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22118400, 22122496) 0 0 00000080:00200000:1.0:1789760772.268317:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760772.268334:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5400, 5400] 00020000:00200000:1.0:1789760772.268343:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22118400 chunk: [22118400, 22122496) 22122496, 4096 00020000:00200000:1.0:1789760772.268347:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760772.268368:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5400, 5400] 00000008:00010000:1.0:1789760772.268407:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760772.268414:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760772.268425:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22118400, 22122496) append 0 append lockpos 0 00000080:00200000:1.0:1789760772.268439:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22118400=0x1518000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760772.268501:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760772.268504:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22122496 00000020:00010000:1.0:1789760772.268512:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760772.268516:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.268520:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.268523:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760772.268526:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760772.268536:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760772.268544:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760772.268567:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760772.270605:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 22118400<>22122496 00000080:00200000:2.0:1789760772.323305:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5400, 5400] 00000080:00200000:2.0:1789760772.323324:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.323330:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.323347:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22122496 i_blocks 45063 00000020:00200000:2.0:1789760772.323351:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.323361:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.323488:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22122496, count: 4096 00000080:00200000:2.0:1789760772.323496:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22122496, bytes: 4096 00000080:00200000:2.0:1789760772.323507:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.323516:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.323519:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22122496, 22126592) 0 0 00000080:00200000:2.0:1789760772.323524:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.323543:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5401, 5401] 00020000:00200000:2.0:1789760772.323554:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22122496 chunk: [22122496, 22126592) 22126592, 4096 00020000:00200000:2.0:1789760772.323560:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.323585:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5401, 5401] 00000008:00010000:2.0:1789760772.323603:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.323609:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.323620:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22122496, 22126592) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.323635:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22122496=0x1519000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.323776:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.323779:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22126592 00000020:00010000:2.0:1789760772.323853:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.323858:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.323862:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.323866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.323869:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.323877:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.323883:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.323962:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760772.324072:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 22122496<>22126592 00000080:00200000:2.0:1789760772.370417:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5401, 5401] 00000080:00200000:2.0:1789760772.370435:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.370441:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.370458:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22126592 i_blocks 45063 00000020:00200000:2.0:1789760772.370463:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.370473:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.370508:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22126592, count: 4096 00000080:00200000:2.0:1789760772.370516:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22126592, bytes: 4096 00000080:00200000:2.0:1789760772.370527:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.370536:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.370539:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22126592, 22130688) 0 0 00000080:00200000:2.0:1789760772.370543:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.370689:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5402, 5402] 00020000:00200000:2.0:1789760772.370698:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22126592 chunk: [22126592, 22130688) 22130688, 4096 00020000:00200000:2.0:1789760772.370703:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.370724:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5402, 5402] 00000008:00010000:2.0:1789760772.370742:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.370747:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.370768:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22126592, 22130688) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.370782:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22126592=0x151a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.370852:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.370855:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22130688 00000020:00010000:2.0:1789760772.370864:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.370867:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.370872:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.370875:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.370878:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.370886:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.370892:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.371019:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760772.371121:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 22126592<>22130688 00000080:00200000:2.0:1789760772.400551:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5402, 5402] 00000080:00200000:2.0:1789760772.400568:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.400574:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.400591:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22130688 i_blocks 45063 00000020:00200000:2.0:1789760772.400595:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.400604:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.400638:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22130688, count: 4096 00000080:00200000:2.0:1789760772.400646:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22130688, bytes: 4096 00000080:00200000:2.0:1789760772.400655:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.400664:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.400666:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22130688, 22134784) 0 0 00000080:00200000:2.0:1789760772.400671:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.400687:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5403, 5403] 00020000:00200000:2.0:1789760772.400696:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22130688 chunk: [22130688, 22134784) 22134784, 4096 00020000:00200000:2.0:1789760772.400701:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.400721:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5403, 5403] 00000008:00010000:2.0:1789760772.400740:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.400746:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.400766:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22130688, 22134784) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.400780:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22130688=0x151b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.401015:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.401018:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22134784 00000020:00010000:2.0:1789760772.401028:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.401032:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.401037:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.401040:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.401043:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.401051:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.401058:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.401075:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760772.401178:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 22130688<>22134784 00000080:00200000:2.0:1789760772.430932:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5403, 5403] 00000080:00200000:2.0:1789760772.430949:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.430954:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.430969:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22134784 i_blocks 45063 00000020:00200000:2.0:1789760772.430972:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.430982:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.431014:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22134784, count: 4096 00000080:00200000:2.0:1789760772.431021:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22134784, bytes: 4096 00000080:00200000:2.0:1789760772.431033:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.431042:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.431044:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22134784, 22138880) 0 0 00000080:00200000:2.0:1789760772.431048:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.431086:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5404, 5404] 00020000:00200000:2.0:1789760772.431095:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22134784 chunk: [22134784, 22138880) 22138880, 4096 00020000:00200000:2.0:1789760772.431099:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.431120:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5404, 5404] 00000008:00010000:2.0:1789760772.431138:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.431144:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.431155:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22134784, 22138880) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.431169:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22134784=0x151c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.431305:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.431309:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22138880 00000020:00010000:2.0:1789760772.431320:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.431324:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.431328:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.431332:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.431336:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.431345:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.431352:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.431371:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760772.431516:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 22134784<>22138880 00000080:00200000:2.0:1789760772.459744:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5404, 5404] 00000080:00200000:2.0:1789760772.459781:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.459788:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.459803:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22138880 i_blocks 45063 00000020:00200000:2.0:1789760772.459807:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.459817:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.459850:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22138880, count: 4096 00000080:00200000:2.0:1789760772.459858:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22138880, bytes: 4096 00000080:00200000:2.0:1789760772.459868:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.459877:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.460220:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22138880, 22142976) 0 0 00000080:00200000:2.0:1789760772.460229:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.460247:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5405, 5405] 00020000:00200000:2.0:1789760772.460319:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22138880 chunk: [22138880, 22142976) 22142976, 4096 00020000:00200000:2.0:1789760772.460324:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.460346:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5405, 5405] 00000008:00010000:2.0:1789760772.460364:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.460370:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.460410:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22138880, 22142976) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.460426:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22138880=0x151d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.460499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.460502:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22142976 00000020:00010000:2.0:1789760772.460511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.460514:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.460518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.460522:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.460525:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.460534:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.460541:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.460562:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760772.460671:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 22138880<>22142976 00000080:00200000:2.0:1789760772.489301:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5405, 5405] 00000080:00200000:2.0:1789760772.489319:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.489325:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.489341:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22142976 i_blocks 45063 00000020:00200000:2.0:1789760772.489345:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.489354:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.489437:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22142976, count: 4096 00000080:00200000:2.0:1789760772.489445:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22142976, bytes: 4096 00000080:00200000:2.0:1789760772.489455:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.489465:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.489467:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22142976, 22147072) 0 0 00000080:00200000:2.0:1789760772.489472:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.489490:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5406, 5406] 00020000:00200000:2.0:1789760772.489500:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22142976 chunk: [22142976, 22147072) 22147072, 4096 00020000:00200000:2.0:1789760772.489505:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.489532:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5406, 5406] 00000008:00010000:2.0:1789760772.489558:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.489564:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.489576:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22142976, 22147072) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.489600:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22142976=0x151e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.489664:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.489667:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22147072 00000020:00010000:2.0:1789760772.489675:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.489678:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.489682:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.489686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.489688:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.489695:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.489701:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.489718:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760772.490002:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022d80 - object 0x280000401:3 offset 22142976<>22147072 00000080:00200000:2.0:1789760772.521323:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5406, 5406] 00000080:00200000:2.0:1789760772.521341:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.521350:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.521368:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22147072 i_blocks 45063 00000020:00200000:2.0:1789760772.521371:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.521407:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.521444:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22147072, count: 4096 00000080:00200000:2.0:1789760772.521451:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22147072, bytes: 4096 00000080:00200000:2.0:1789760772.521463:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.521473:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.521476:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22147072, 22151168) 0 0 00000080:00200000:2.0:1789760772.521480:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.521498:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5407, 5407] 00020000:00200000:2.0:1789760772.521506:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22147072 chunk: [22147072, 22151168) 22151168, 4096 00020000:00200000:2.0:1789760772.521544:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.521568:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5407, 5407] 00000008:00010000:2.0:1789760772.521586:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.521591:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.521602:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22147072, 22151168) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.521617:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22147072=0x151f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.521676:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.521679:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22151168 00000020:00010000:2.0:1789760772.521687:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.521691:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.521695:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.521700:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.521703:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.521711:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.521718:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.521735:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760772.522159:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 22147072<>22151168 00000080:00200000:2.0:1789760772.549012:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5407, 5407] 00000080:00200000:2.0:1789760772.549026:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.549032:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.549045:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22151168 i_blocks 45063 00000020:00200000:2.0:1789760772.549048:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.549056:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.549086:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22151168, count: 4096 00000080:00200000:2.0:1789760772.549091:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22151168, bytes: 4096 00000080:00200000:2.0:1789760772.549101:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.549109:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.549112:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22151168, 22155264) 0 0 00000080:00200000:2.0:1789760772.549116:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.549133:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5408, 5408] 00020000:00200000:2.0:1789760772.549141:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22151168 chunk: [22151168, 22155264) 22155264, 4096 00020000:00200000:2.0:1789760772.549145:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.549167:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5408, 5408] 00000008:00010000:2.0:1789760772.549186:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.549258:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.549273:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22151168, 22155264) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.549346:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22151168=0x1520000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.549456:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.549459:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22155264 00000020:00010000:2.0:1789760772.549468:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.549472:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.549476:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.549480:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.549483:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.549491:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.549499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.549517:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760772.549625:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 22151168<>22155264 00000080:00200000:2.0:1789760772.575351:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5408, 5408] 00000080:00200000:2.0:1789760772.575367:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.575373:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.575411:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22155264 i_blocks 45063 00000020:00200000:2.0:1789760772.575415:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.575425:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.575464:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22155264, count: 4096 00000080:00200000:2.0:1789760772.575472:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22155264, bytes: 4096 00000080:00200000:2.0:1789760772.575484:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.575493:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.575495:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22155264, 22159360) 0 0 00000080:00200000:2.0:1789760772.575500:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.575516:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5409, 5409] 00020000:00200000:2.0:1789760772.575525:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22155264 chunk: [22155264, 22159360) 22159360, 4096 00020000:00200000:2.0:1789760772.575529:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.575550:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5409, 5409] 00000008:00010000:2.0:1789760772.575571:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.575576:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.575589:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22155264, 22159360) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.575604:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22155264=0x1521000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.575666:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.575668:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22159360 00000020:00010000:2.0:1789760772.575676:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.575679:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.575684:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.575687:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.575690:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.575698:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.575704:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.575720:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760772.575965:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 22155264<>22159360 00000080:00200000:2.0:1789760772.637556:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5409, 5409] 00000080:00200000:2.0:1789760772.637639:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.637760:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.637776:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22159360 i_blocks 45063 00000020:00200000:2.0:1789760772.637780:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.637790:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.637823:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22159360, count: 4096 00000080:00200000:2.0:1789760772.637830:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22159360, bytes: 4096 00000080:00200000:2.0:1789760772.637841:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.637849:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.637851:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22159360, 22163456) 0 0 00000080:00200000:2.0:1789760772.637855:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.637872:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5410, 5410] 00020000:00200000:2.0:1789760772.637880:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22159360 chunk: [22159360, 22163456) 22163456, 4096 00020000:00200000:2.0:1789760772.637885:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.638808:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5410, 5410] 00000008:00010000:2.0:1789760772.638834:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.638840:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.638855:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22159360, 22163456) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.638872:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22159360=0x1522000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.638939:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.638943:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22163456 00000020:00010000:2.0:1789760772.638951:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.638955:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.638959:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.638963:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.638966:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.638975:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.638983:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.639007:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760772.639114:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 22159360<>22163456 00000080:00200000:2.0:1789760772.676040:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5410, 5410] 00000080:00200000:2.0:1789760772.676059:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.676065:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.676081:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22163456 i_blocks 45063 00000020:00200000:2.0:1789760772.676084:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.676094:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.676130:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22163456, count: 4096 00000080:00200000:2.0:1789760772.676138:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22163456, bytes: 4096 00000080:00200000:2.0:1789760772.676152:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.676160:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.676162:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22163456, 22167552) 0 0 00000080:00200000:2.0:1789760772.676167:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.676183:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5411, 5411] 00020000:00200000:2.0:1789760772.676192:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22163456 chunk: [22163456, 22167552) 22167552, 4096 00020000:00200000:2.0:1789760772.676267:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.676292:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5411, 5411] 00000008:00010000:2.0:1789760772.676313:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.676320:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.676413:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22163456, 22167552) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.676428:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22163456=0x1523000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.676498:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.676501:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22167552 00000020:00010000:2.0:1789760772.676512:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.676516:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.676520:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.676524:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.676527:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.676535:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.676542:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.676562:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760772.676687:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 22163456<>22167552 00000080:00200000:2.0:1789760772.706975:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5411, 5411] 00000080:00200000:2.0:1789760772.706994:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760772.707001:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760772.707018:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22167552 i_blocks 45063 00000020:00200000:2.0:1789760772.707023:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760772.707034:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760772.707079:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22167552, count: 4096 00000080:00200000:2.0:1789760772.707088:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22167552, bytes: 4096 00000080:00200000:2.0:1789760772.707098:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760772.707107:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760772.707109:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22167552, 22171648) 0 0 00000080:00200000:2.0:1789760772.707113:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760772.707130:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5412, 5412] 00020000:00200000:2.0:1789760772.707137:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22167552 chunk: [22167552, 22171648) 22171648, 4096 00020000:00200000:2.0:1789760772.707141:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760772.707158:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5412, 5412] 00000008:00010000:2.0:1789760772.707175:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760772.707180:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760772.707190:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22167552, 22171648) append 0 append lockpos 0 00000080:00200000:2.0:1789760772.707261:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22167552=0x1524000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760772.707310:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760772.707312:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22171648 00000020:00010000:2.0:1789760772.707319:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760772.707321:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.707324:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760772.707326:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760772.707328:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760772.707334:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760772.707339:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760772.707355:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760772.707524:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 22167552<>22171648 00000100:00100000:1.0:1789760772.801804:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972733 gen 3 00000008:00010000:1.0:1789760772.801852:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760772.801858:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760772.801868:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760772.801898:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760772.801925:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507327488:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760772.839705:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5414, 5414] 00000080:00200000:1.0:1789760772.839718:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760772.839724:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760772.839735:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22179840 i_blocks 45063 00000020:00200000:1.0:1789760772.839738:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760772.839747:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760772.839781:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22179840, count: 4096 00000080:00200000:1.0:1789760772.839787:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22179840, bytes: 4096 00000080:00200000:1.0:1789760772.839795:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760772.839802:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760772.839804:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22179840, 22183936) 0 0 00000080:00200000:1.0:1789760772.839808:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760772.839820:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5415, 5415] 00020000:00200000:1.0:1789760772.839827:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22179840 chunk: [22179840, 22183936) 22183936, 4096 00020000:00200000:1.0:1789760772.839830:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760772.839845:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5415, 5415] 00000008:00010000:1.0:1789760772.839859:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760772.839863:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760772.839872:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22179840, 22183936) append 0 append lockpos 0 00000080:00200000:1.0:1789760772.839881:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22179840=0x1527000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760772.839927:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760772.839930:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22183936 00000020:00010000:1.0:1789760772.839936:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760772.839940:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.839943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760772.839946:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760772.839949:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760772.839956:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760772.839962:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760772.839976:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760772.840054:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 22179840<>22183936 00000100:00100000:1.0:1789760772.840096:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760772.840111:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507327744:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760772.859944:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507328000:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760772.870555:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507328000:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760772.870569:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507328256:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760772.879259:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760772.879294:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507328512:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760772.931564:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972736 gen 3 00000008:00010000:2.0:1789760772.931622:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c900 00000020:00200000:2.0:1789760772.931632:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760772.931648:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760772.931682:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c900 00000100:00100000:2.0:1789760772.931718:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507328512:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760773.067293:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507328896:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760773.167696:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972739 gen 3 00000008:00010000:2.0:1789760773.167734:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000020:00200000:2.0:1789760773.167741:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760773.167761:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760773.167781:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000100:00100000:2.0:1789760773.167805:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507328896:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760773.169513:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507329024:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760773.204118:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972740 gen 3 00000008:00010000:2.0:1789760773.204158:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5ca80 00000020:00200000:2.0:1789760773.204165:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760773.204176:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760773.204198:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5ca80 00000100:00100000:2.0:1789760773.204223:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507329024:192.168.203.141@tcp:4:dd.0 00000080:00200008:2.0:1789760773.208166:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22204416, count: 4096 00000080:00200000:2.0:1789760773.208191:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22204416, bytes: 4096 00000080:00200000:2.0:1789760773.208277:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760773.208288:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760773.208290:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22204416, 22208512) 0 0 00000080:00200000:2.0:1789760773.208297:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760773.208317:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5421, 5421] 00020000:00200000:2.0:1789760773.208327:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22204416 chunk: [22204416, 22208512) 22208512, 4096 00020000:00200000:2.0:1789760773.208333:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760773.208413:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5421, 5421] 00000008:00010000:2.0:1789760773.208432:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760773.208438:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760773.208450:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22204416, 22208512) append 0 append lockpos 0 00000080:00200000:2.0:1789760773.208465:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22204416=0x152d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760773.208538:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760773.208542:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22208512 00000020:00010000:2.0:1789760773.208550:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760773.208554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760773.208557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760773.208560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760773.208563:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760773.208577:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760773.208586:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760773.208633:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760773.209293:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 22204416<>22208512 00000100:00100000:1.0:1789760773.253884:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507329280:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760773.301801:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5422, 5422] 00000080:00200000:2.0:1789760773.301817:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760773.301824:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760773.301850:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22212608 i_blocks 45063 00000020:00200000:2.0:1789760773.301854:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760773.301865:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760773.301905:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22212608, count: 4096 00000080:00200000:2.0:1789760773.301913:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22212608, bytes: 4096 00000080:00200000:2.0:1789760773.301925:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760773.301934:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760773.301937:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22212608, 22216704) 0 0 00000080:00200000:2.0:1789760773.301942:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760773.301961:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5423, 5423] 00020000:00200000:2.0:1789760773.301970:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22212608 chunk: [22212608, 22216704) 22216704, 4096 00020000:00200000:2.0:1789760773.301975:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760773.301996:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5423, 5423] 00000008:00010000:2.0:1789760773.302599:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760773.302606:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760773.302621:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22212608, 22216704) append 0 append lockpos 0 00000080:00200000:2.0:1789760773.302635:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22212608=0x152f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760773.302700:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760773.302703:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22216704 00000020:00010000:2.0:1789760773.302711:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760773.302715:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760773.302719:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760773.302723:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760773.302726:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760773.302734:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760773.302741:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760773.302770:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760773.302881:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 22212608<>22216704 00000100:00100000:1.0:1789760773.374843:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507329536:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760773.428988:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5424, 5424] 00000080:00200000:1.0:1789760773.429003:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.429009:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.429026:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22220800 i_blocks 45063 00000020:00200000:1.0:1789760773.429030:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.429040:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.429075:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22220800, count: 4096 00000080:00200000:1.0:1789760773.429083:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22220800, bytes: 4096 00000080:00200000:1.0:1789760773.429094:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.429104:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.429106:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22220800, 22224896) 0 0 00000080:00200000:1.0:1789760773.429111:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.429127:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5425, 5425] 00020000:00200000:1.0:1789760773.429137:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22220800 chunk: [22220800, 22224896) 22224896, 4096 00020000:00200000:1.0:1789760773.429142:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.429161:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5425, 5425] 00000008:00010000:1.0:1789760773.429182:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.429199:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.429210:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22220800, 22224896) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.429224:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22220800=0x1531000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.429279:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.429282:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22224896 00000020:00010000:1.0:1789760773.429290:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.429294:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.429297:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.429301:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.429303:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.429311:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.429318:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.429334:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760773.429481:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 22220800<>22224896 00000080:00200000:1.0:1789760773.472667:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5425, 5425] 00000080:00200000:1.0:1789760773.472685:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.472690:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.472705:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22224896 i_blocks 45063 00000020:00200000:1.0:1789760773.472709:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.472719:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.472976:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22224896, count: 4096 00000080:00200000:1.0:1789760773.472986:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22224896, bytes: 4096 00000080:00200000:1.0:1789760773.472997:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.473008:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.473010:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22224896, 22228992) 0 0 00000080:00200000:1.0:1789760773.473015:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.473032:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5426, 5426] 00020000:00200000:1.0:1789760773.473041:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22224896 chunk: [22224896, 22228992) 22228992, 4096 00020000:00200000:1.0:1789760773.473047:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.473067:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5426, 5426] 00000008:00010000:1.0:1789760773.473085:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.473091:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.473103:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22224896, 22228992) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.473117:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22224896=0x1532000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.473178:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.473580:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22228992 00000020:00010000:1.0:1789760773.473592:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.473596:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.473599:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.473603:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.473606:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.473616:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.473623:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.473649:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760773.473769:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 22224896<>22228992 00000080:00200000:1.0:1789760773.509355:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5426, 5426] 00000080:00200000:1.0:1789760773.509370:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.509407:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.509422:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22228992 i_blocks 45063 00000020:00200000:1.0:1789760773.509426:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.509436:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.509471:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22228992, count: 4096 00000080:00200000:1.0:1789760773.509479:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22228992, bytes: 4096 00000080:00200000:1.0:1789760773.509488:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.509497:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.509499:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22228992, 22233088) 0 0 00000080:00200000:1.0:1789760773.509504:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.509521:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5427, 5427] 00020000:00200000:1.0:1789760773.509530:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22228992 chunk: [22228992, 22233088) 22233088, 4096 00020000:00200000:1.0:1789760773.509535:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.509553:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5427, 5427] 00000008:00010000:1.0:1789760773.509569:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.509574:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.509583:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22228992, 22233088) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.509595:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22228992=0x1533000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.509639:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.509641:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22233088 00000020:00010000:1.0:1789760773.509648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.509650:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.509652:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.509655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.509657:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.509663:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.509667:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.509681:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760773.509894:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 22228992<>22233088 00000080:00200000:1.0:1789760773.562580:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5427, 5427] 00000080:00200000:1.0:1789760773.562599:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.562605:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.562620:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22233088 i_blocks 45063 00000020:00200000:1.0:1789760773.562624:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.562634:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.562675:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22233088, count: 4096 00000080:00200000:1.0:1789760773.562684:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22233088, bytes: 4096 00000080:00200000:1.0:1789760773.562698:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.562708:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.562710:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22233088, 22237184) 0 0 00000080:00200000:1.0:1789760773.562714:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.562732:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5428, 5428] 00020000:00200000:1.0:1789760773.562740:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22233088 chunk: [22233088, 22237184) 22237184, 4096 00020000:00200000:1.0:1789760773.563569:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.563598:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5428, 5428] 00000008:00010000:1.0:1789760773.563622:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.563627:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.563641:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22233088, 22237184) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.563657:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22233088=0x1534000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.563724:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.563727:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22237184 00000020:00010000:1.0:1789760773.563736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.563739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.563743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.563951:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.563955:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.563968:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.563978:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.564006:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760773.564181:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22233088<>22237184 00000080:00200000:1.0:1789760773.621987:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5428, 5428] 00000080:00200000:1.0:1789760773.622007:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.622013:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.622029:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22237184 i_blocks 45063 00000020:00200000:1.0:1789760773.622033:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.622044:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.622076:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22237184, count: 4096 00000080:00200000:1.0:1789760773.622085:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22237184, bytes: 4096 00000080:00200000:1.0:1789760773.622095:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.622105:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.622107:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22237184, 22241280) 0 0 00000080:00200000:1.0:1789760773.622112:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.622130:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5429, 5429] 00020000:00200000:1.0:1789760773.622139:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22237184 chunk: [22237184, 22241280) 22241280, 4096 00020000:00200000:1.0:1789760773.622144:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.622163:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5429, 5429] 00000008:00010000:1.0:1789760773.622182:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.622200:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.622213:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22237184, 22241280) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.622228:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22237184=0x1535000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.622526:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.622529:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22241280 00000020:00010000:1.0:1789760773.622538:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.622542:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.622546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.622550:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.622553:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.622561:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.622569:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.622584:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760773.622685:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 22237184<>22241280 00000080:00200000:1.0:1789760773.691596:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5429, 5429] 00000080:00200000:1.0:1789760773.691609:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.691614:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.691626:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22241280 i_blocks 45063 00000020:00200000:1.0:1789760773.691629:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.691637:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.691662:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22241280, count: 4096 00000080:00200000:1.0:1789760773.691669:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22241280, bytes: 4096 00000080:00200000:1.0:1789760773.691676:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.691685:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.691687:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22241280, 22245376) 0 0 00000080:00200000:1.0:1789760773.691691:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.691703:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5430, 5430] 00020000:00200000:1.0:1789760773.691709:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22241280 chunk: [22241280, 22245376) 22245376, 4096 00020000:00200000:1.0:1789760773.691712:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.691728:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5430, 5430] 00000008:00010000:1.0:1789760773.691838:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.691845:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.691856:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22241280, 22245376) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.691867:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22241280=0x1536000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.691910:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.691913:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22245376 00000020:00010000:1.0:1789760773.691918:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.691921:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.691924:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.691927:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.691929:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.691936:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.691942:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.691958:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760773.692034:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 22241280<>22245376 00000080:00200000:1.0:1789760773.725436:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5430, 5430] 00000080:00200000:1.0:1789760773.725451:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.725457:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.725471:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22245376 i_blocks 45063 00000020:00200000:1.0:1789760773.725475:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.725484:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.725514:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22245376, count: 4096 00000080:00200000:1.0:1789760773.725521:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22245376, bytes: 4096 00000080:00200000:1.0:1789760773.725530:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.725539:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.725541:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22245376, 22249472) 0 0 00000080:00200000:1.0:1789760773.725546:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.725561:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5431, 5431] 00020000:00200000:1.0:1789760773.725569:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22245376 chunk: [22245376, 22249472) 22249472, 4096 00020000:00200000:1.0:1789760773.725573:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.725593:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5431, 5431] 00000008:00010000:1.0:1789760773.725610:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.725616:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.725627:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22245376, 22249472) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.725641:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22245376=0x1537000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.725699:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.725703:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22249472 00000020:00010000:1.0:1789760773.725710:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.725713:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.725716:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.725719:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.725723:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.725730:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.725737:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.725764:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760773.725864:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 22245376<>22249472 00000080:00200000:1.0:1789760773.770559:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5431, 5431] 00000080:00200000:1.0:1789760773.770930:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.770938:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.770958:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22249472 i_blocks 45063 00000020:00200000:1.0:1789760773.770967:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.770982:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.771017:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22249472, count: 4096 00000080:00200000:1.0:1789760773.771025:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22249472, bytes: 4096 00000080:00200000:1.0:1789760773.771040:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.771050:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.771052:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22249472, 22253568) 0 0 00000080:00200000:1.0:1789760773.771057:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.771073:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5432, 5432] 00020000:00200000:1.0:1789760773.771080:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22249472 chunk: [22249472, 22253568) 22253568, 4096 00020000:00200000:1.0:1789760773.771085:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.771338:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5432, 5432] 00000008:00010000:1.0:1789760773.771360:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.771366:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.771408:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22249472, 22253568) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.771424:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22249472=0x1538000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.771493:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.771496:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22253568 00000020:00010000:1.0:1789760773.771504:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.771508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.771512:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.771515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.771518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.771543:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.771551:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.771793:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760773.771912:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 22249472<>22253568 00000080:00200000:1.0:1789760773.807805:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5432, 5432] 00000080:00200000:1.0:1789760773.807821:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.807827:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.807842:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22253568 i_blocks 45063 00000020:00200000:1.0:1789760773.807846:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.807856:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.807895:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22253568, count: 4096 00000080:00200000:1.0:1789760773.807902:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22253568, bytes: 4096 00000080:00200000:1.0:1789760773.807912:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.807921:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.807923:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22253568, 22257664) 0 0 00000080:00200000:1.0:1789760773.807928:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.807944:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5433, 5433] 00020000:00200000:1.0:1789760773.807953:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22253568 chunk: [22253568, 22257664) 22257664, 4096 00020000:00200000:1.0:1789760773.807958:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.807977:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5433, 5433] 00000008:00010000:1.0:1789760773.807996:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.808002:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.808014:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22253568, 22257664) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.808028:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22253568=0x1539000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.808085:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.808088:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22257664 00000020:00010000:1.0:1789760773.808096:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.808100:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.808103:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.808107:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.808110:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.808118:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.808125:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.808141:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760773.808305:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 22253568<>22257664 00000080:00200000:1.0:1789760773.867612:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5433, 5433] 00000080:00200000:1.0:1789760773.867643:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.867650:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.867917:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22257664 i_blocks 45063 00000020:00200000:1.0:1789760773.867923:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.868004:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.868045:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22257664, count: 4096 00000080:00200000:1.0:1789760773.868055:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22257664, bytes: 4096 00000080:00200000:1.0:1789760773.868072:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.868083:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.868085:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22257664, 22261760) 0 0 00000080:00200000:1.0:1789760773.868090:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.868282:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5434, 5434] 00020000:00200000:1.0:1789760773.868292:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22257664 chunk: [22257664, 22261760) 22261760, 4096 00020000:00200000:1.0:1789760773.868297:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.868320:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5434, 5434] 00000008:00010000:1.0:1789760773.868340:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.868345:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.868357:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22257664, 22261760) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.868372:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22257664=0x153a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.868472:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.868475:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22261760 00000020:00010000:1.0:1789760773.868484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.868488:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.868491:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.868494:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.868498:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.868507:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.868514:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.868546:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760773.868726:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 22257664<>22261760 00000080:00200000:1.0:1789760773.917389:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5434, 5434] 00000080:00200000:1.0:1789760773.917411:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.917417:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.917434:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22261760 i_blocks 45063 00000020:00200000:1.0:1789760773.917439:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.917453:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.917486:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22261760, count: 4096 00000080:00200000:1.0:1789760773.917494:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22261760, bytes: 4096 00000080:00200000:1.0:1789760773.917508:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.917519:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.917521:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22261760, 22265856) 0 0 00000080:00200000:1.0:1789760773.917526:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.917542:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5435, 5435] 00020000:00200000:1.0:1789760773.917551:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22261760 chunk: [22261760, 22265856) 22265856, 4096 00020000:00200000:1.0:1789760773.917555:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.917576:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5435, 5435] 00000008:00010000:1.0:1789760773.917594:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.917600:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.917612:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22261760, 22265856) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.917626:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22261760=0x153b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.917895:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.917899:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22265856 00000020:00010000:1.0:1789760773.917909:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.917913:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.917916:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.917920:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.917923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.917932:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.917938:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.917971:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760773.918082:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 22261760<>22265856 00000080:00200000:1.0:1789760773.955330:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5435, 5435] 00000080:00200000:1.0:1789760773.955347:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760773.955358:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760773.955374:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22265856 i_blocks 45063 00000020:00200000:1.0:1789760773.955403:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760773.955413:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760773.955447:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22265856, count: 4096 00000080:00200000:1.0:1789760773.955455:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22265856, bytes: 4096 00000080:00200000:1.0:1789760773.955464:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760773.955474:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760773.955476:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22265856, 22269952) 0 0 00000080:00200000:1.0:1789760773.955481:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760773.955497:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5436, 5436] 00020000:00200000:1.0:1789760773.955506:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22265856 chunk: [22265856, 22269952) 22269952, 4096 00020000:00200000:1.0:1789760773.955511:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760773.955531:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5436, 5436] 00000008:00010000:1.0:1789760773.955550:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760773.955556:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760773.955567:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22265856, 22269952) append 0 append lockpos 0 00000080:00200000:1.0:1789760773.955583:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22265856=0x153c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760773.955642:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760773.955645:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22269952 00000020:00010000:1.0:1789760773.955653:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760773.955656:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.955659:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760773.955662:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760773.955665:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760773.955673:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760773.955681:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760773.955700:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760773.955815:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 22265856<>22269952 00000080:00200000:1.0:1789760774.007315:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5436, 5436] 00000080:00200000:1.0:1789760774.007338:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.007346:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.007366:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22269952 i_blocks 45063 00000020:00200000:1.0:1789760774.007371:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.007414:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.007451:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22269952, count: 4096 00000080:00200000:1.0:1789760774.007458:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22269952, bytes: 4096 00000080:00200000:1.0:1789760774.007471:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.007483:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.007485:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22269952, 22274048) 0 0 00000080:00200000:1.0:1789760774.007490:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.007507:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5437, 5437] 00020000:00200000:1.0:1789760774.007516:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22269952 chunk: [22269952, 22274048) 22274048, 4096 00020000:00200000:1.0:1789760774.007520:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.007541:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5437, 5437] 00000008:00010000:1.0:1789760774.007557:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.007563:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.007573:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22269952, 22274048) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.007588:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22269952=0x153d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.007644:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.007647:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22274048 00000020:00010000:1.0:1789760774.007657:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.007661:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.007664:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.007668:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.007672:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.007681:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.007690:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.008116:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760774.009614:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 22269952<>22274048 00000080:00200000:1.0:1789760774.064762:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5437, 5437] 00000080:00200000:1.0:1789760774.064780:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.064787:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.064805:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22274048 i_blocks 45063 00000020:00200000:1.0:1789760774.064810:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.064821:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.064859:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22274048, count: 4096 00000080:00200000:1.0:1789760774.064867:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22274048, bytes: 4096 00000080:00200000:1.0:1789760774.064882:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.064893:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.064896:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22274048, 22278144) 0 0 00000080:00200000:1.0:1789760774.064902:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.064922:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5438, 5438] 00020000:00200000:1.0:1789760774.064932:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22274048 chunk: [22274048, 22278144) 22278144, 4096 00020000:00200000:1.0:1789760774.064937:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.064960:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5438, 5438] 00000008:00010000:1.0:1789760774.064982:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.064989:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.065002:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22274048, 22278144) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.065016:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22274048=0x153e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.065078:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.065081:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22278144 00000020:00010000:1.0:1789760774.065089:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.065093:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.065096:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.065101:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.065105:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.065114:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.065121:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.065142:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760774.065265:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 22274048<>22278144 00000080:00200000:1.0:1789760774.111106:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5438, 5438] 00000080:00200000:1.0:1789760774.111122:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.111128:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.111144:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22278144 i_blocks 45063 00000020:00200000:1.0:1789760774.111147:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.111158:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.111256:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22278144, count: 4096 00000080:00200000:1.0:1789760774.111264:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22278144, bytes: 4096 00000080:00200000:1.0:1789760774.111274:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.111285:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.111287:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22278144, 22282240) 0 0 00000080:00200000:1.0:1789760774.111291:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.111309:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5439, 5439] 00020000:00200000:1.0:1789760774.111317:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22278144 chunk: [22278144, 22282240) 22282240, 4096 00020000:00200000:1.0:1789760774.111321:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.111340:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5439, 5439] 00000008:00010000:1.0:1789760774.111359:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.111364:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.111411:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22278144, 22282240) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.111425:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22278144=0x153f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.111480:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.111484:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22282240 00000020:00010000:1.0:1789760774.111492:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.111495:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.111498:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.111502:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.111505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.111514:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.111521:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.111539:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760774.111643:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 22278144<>22282240 00000080:00200000:1.0:1789760774.143970:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5439, 5439] 00000080:00200000:1.0:1789760774.143990:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.143997:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.144013:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22282240 i_blocks 45063 00000020:00200000:1.0:1789760774.144017:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.144027:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.144061:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22282240, count: 4096 00000080:00200000:1.0:1789760774.144070:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22282240, bytes: 4096 00000080:00200000:1.0:1789760774.144084:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.144095:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.144097:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22282240, 22286336) 0 0 00000080:00200000:1.0:1789760774.144102:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.144118:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5440, 5440] 00020000:00200000:1.0:1789760774.144127:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22282240 chunk: [22282240, 22286336) 22286336, 4096 00020000:00200000:1.0:1789760774.144131:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.144151:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5440, 5440] 00000008:00010000:1.0:1789760774.144168:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.144174:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.144186:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22282240, 22286336) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.144201:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22282240=0x1540000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.144258:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.144262:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22286336 00000020:00010000:1.0:1789760774.144271:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.144274:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.144277:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.144281:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.144284:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.144292:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.144308:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.144326:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760774.144478:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22282240<>22286336 00000080:00200000:1.0:1789760774.199721:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5440, 5440] 00000080:00200000:1.0:1789760774.199737:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.199753:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.199769:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22286336 i_blocks 45063 00000020:00200000:1.0:1789760774.199773:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.199784:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.199819:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22286336, count: 4096 00000080:00200000:1.0:1789760774.199826:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22286336, bytes: 4096 00000080:00200000:1.0:1789760774.199837:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.199846:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.199849:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22286336, 22290432) 0 0 00000080:00200000:1.0:1789760774.199853:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.199871:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5441, 5441] 00020000:00200000:1.0:1789760774.199881:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22286336 chunk: [22286336, 22290432) 22290432, 4096 00020000:00200000:1.0:1789760774.199886:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.199906:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5441, 5441] 00000008:00010000:1.0:1789760774.199926:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.199942:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.199955:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22286336, 22290432) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.199970:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22286336=0x1541000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.200037:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.200041:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22290432 00000020:00010000:1.0:1789760774.200050:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.200053:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.200056:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.200060:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.200062:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.200071:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.200079:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.200107:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760774.200443:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 22286336<>22290432 00000080:00200000:1.0:1789760774.243112:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5441, 5441] 00000080:00200000:1.0:1789760774.243131:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.243138:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.243156:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22290432 i_blocks 45063 00000020:00200000:1.0:1789760774.243161:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.243174:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.243217:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22290432, count: 4096 00000080:00200000:1.0:1789760774.243224:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22290432, bytes: 4096 00000080:00200000:1.0:1789760774.243235:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.243246:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.243248:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22290432, 22294528) 0 0 00000080:00200000:1.0:1789760774.243253:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.243269:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5442, 5442] 00020000:00200000:1.0:1789760774.243277:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22290432 chunk: [22290432, 22294528) 22294528, 4096 00020000:00200000:1.0:1789760774.243282:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.243303:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5442, 5442] 00000008:00010000:1.0:1789760774.243323:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.243329:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.243340:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22290432, 22294528) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.243354:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22290432=0x1542000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.243439:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.243442:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22294528 00000020:00010000:1.0:1789760774.243450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.243453:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.243456:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.243459:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.243462:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.243471:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.243477:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.243493:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760774.243588:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 22290432<>22294528 00000080:00200000:1.0:1789760774.278238:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5442, 5442] 00000080:00200000:1.0:1789760774.278255:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.278262:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.278279:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22294528 i_blocks 45063 00000020:00200000:1.0:1789760774.278282:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.278294:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.278407:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22294528, count: 4096 00000080:00200000:1.0:1789760774.278415:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22294528, bytes: 4096 00000080:00200000:1.0:1789760774.278428:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.278438:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.278441:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22294528, 22298624) 0 0 00000080:00200000:1.0:1789760774.278446:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.278464:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5443, 5443] 00020000:00200000:1.0:1789760774.278474:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22294528 chunk: [22294528, 22298624) 22298624, 4096 00020000:00200000:1.0:1789760774.278479:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.278500:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5443, 5443] 00000008:00010000:1.0:1789760774.278596:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.278603:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.278615:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22294528, 22298624) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.278637:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22294528=0x1543000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.278700:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.278704:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22298624 00000020:00010000:1.0:1789760774.278712:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.278716:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.278719:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.278722:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.278725:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.278860:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.278869:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.278891:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760774.279076:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 22294528<>22298624 00000080:00200000:1.0:1789760774.312632:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5443, 5443] 00000080:00200000:1.0:1789760774.312649:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.312655:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.312671:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22298624 i_blocks 45063 00000020:00200000:1.0:1789760774.312675:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.312684:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.312718:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22298624, count: 4096 00000080:00200000:1.0:1789760774.312726:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22298624, bytes: 4096 00000080:00200000:1.0:1789760774.312736:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.312759:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.312762:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22298624, 22302720) 0 0 00000080:00200000:1.0:1789760774.312768:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.312798:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5444, 5444] 00020000:00200000:1.0:1789760774.312807:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22298624 chunk: [22298624, 22302720) 22302720, 4096 00020000:00200000:1.0:1789760774.312812:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.312832:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5444, 5444] 00000008:00010000:1.0:1789760774.312848:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.312855:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.312866:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22298624, 22302720) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.312880:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22298624=0x1544000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.312943:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.312946:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22302720 00000020:00010000:1.0:1789760774.312954:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.312957:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.312960:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.312964:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.312967:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.312975:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.312981:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.312999:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760774.313102:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 22298624<>22302720 00000080:00200000:1.0:1789760774.350284:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5444, 5444] 00000080:00200000:1.0:1789760774.350299:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.350306:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.350321:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22302720 i_blocks 45063 00000020:00200000:1.0:1789760774.350325:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.350334:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.350369:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22302720, count: 4096 00000080:00200000:1.0:1789760774.350427:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22302720, bytes: 4096 00000080:00200000:1.0:1789760774.350437:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.350446:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.350448:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22302720, 22306816) 0 0 00000080:00200000:1.0:1789760774.350453:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.350470:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5445, 5445] 00020000:00200000:1.0:1789760774.350479:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22302720 chunk: [22302720, 22306816) 22306816, 4096 00020000:00200000:1.0:1789760774.350483:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.350504:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5445, 5445] 00000008:00010000:1.0:1789760774.350522:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.350528:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.350540:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22302720, 22306816) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.350555:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22302720=0x1545000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.350610:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.350613:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22306816 00000020:00010000:1.0:1789760774.350622:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.350626:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.350628:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.350632:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.350635:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.350643:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.350649:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.350666:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760774.350780:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 22302720<>22306816 00000080:00200000:1.0:1789760774.391936:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5445, 5445] 00000080:00200000:1.0:1789760774.391953:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.391959:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.391974:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22306816 i_blocks 45063 00000020:00200000:1.0:1789760774.391978:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.391988:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.392025:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22306816, count: 4096 00000080:00200000:1.0:1789760774.392523:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22306816, bytes: 4096 00000080:00200000:1.0:1789760774.392541:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.392552:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.392555:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22306816, 22310912) 0 0 00000080:00200000:1.0:1789760774.392560:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.392579:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5446, 5446] 00020000:00200000:1.0:1789760774.392590:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22306816 chunk: [22306816, 22310912) 22310912, 4096 00020000:00200000:1.0:1789760774.392595:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.392620:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5446, 5446] 00000008:00010000:1.0:1789760774.392643:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.392649:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.392662:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22306816, 22310912) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.392677:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22306816=0x1546000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.392755:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.392758:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22310912 00000020:00010000:1.0:1789760774.392767:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.392771:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.392774:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.392777:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.392781:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.392791:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.392799:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.392820:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760774.392941:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 22306816<>22310912 00000080:00200000:1.0:1789760774.442564:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5446, 5446] 00000080:00200000:1.0:1789760774.442599:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.442607:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.442627:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22310912 i_blocks 45063 00000020:00200000:1.0:1789760774.442632:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.442649:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.442688:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22310912, count: 4096 00000080:00200000:1.0:1789760774.442698:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22310912, bytes: 4096 00000080:00200000:1.0:1789760774.442713:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.442723:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.442725:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22310912, 22315008) 0 0 00000080:00200000:1.0:1789760774.442731:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.442761:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5447, 5447] 00020000:00200000:1.0:1789760774.442770:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22310912 chunk: [22310912, 22315008) 22315008, 4096 00020000:00200000:1.0:1789760774.442774:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.442796:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5447, 5447] 00000008:00010000:1.0:1789760774.442816:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.442822:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.442833:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22310912, 22315008) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.442849:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22310912=0x1547000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.442924:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.442927:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22315008 00000020:00010000:1.0:1789760774.442936:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.442940:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.442943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.442947:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.442950:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.442959:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.442967:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.443006:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760774.443421:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 22310912<>22315008 00000100:00100000:1.0:1789760774.443501:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507332480:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760774.483220:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5447, 5447] 00000080:00200000:1.0:1789760774.483237:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.483243:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.483270:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22315008 i_blocks 45063 00000020:00200000:1.0:1789760774.483274:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.483285:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.483321:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22315008, count: 4096 00000080:00200000:1.0:1789760774.483329:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22315008, bytes: 4096 00000080:00200000:1.0:1789760774.483338:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.483346:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.483349:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22315008, 22319104) 0 0 00000080:00200000:1.0:1789760774.483353:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.483369:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5448, 5448] 00020000:00200000:1.0:1789760774.483426:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22315008 chunk: [22315008, 22319104) 22319104, 4096 00020000:00200000:1.0:1789760774.483432:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.483452:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5448, 5448] 00000008:00010000:1.0:1789760774.483470:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.483475:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.483488:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22315008, 22319104) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.483501:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22315008=0x1548000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.483558:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.483561:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22319104 00000020:00010000:1.0:1789760774.483579:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.483584:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.483587:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.483591:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.483594:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.483602:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.483609:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.483629:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760774.483731:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 22315008<>22319104 00000080:00200000:1.0:1789760774.519719:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5448, 5448] 00000080:00200000:1.0:1789760774.519734:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.519752:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.519767:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22319104 i_blocks 45063 00000020:00200000:1.0:1789760774.519771:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.519782:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.520329:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22319104, count: 4096 00000080:00200000:1.0:1789760774.520341:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22319104, bytes: 4096 00000080:00200000:1.0:1789760774.520353:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.520361:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.520363:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22319104, 22323200) 0 0 00000080:00200000:1.0:1789760774.520368:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.520428:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5449, 5449] 00020000:00200000:1.0:1789760774.520438:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22319104 chunk: [22319104, 22323200) 22323200, 4096 00020000:00200000:1.0:1789760774.520443:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.520464:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5449, 5449] 00000008:00010000:1.0:1789760774.520481:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.520485:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.520497:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22319104, 22323200) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.520511:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22319104=0x1549000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.520564:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.520567:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22323200 00000020:00010000:1.0:1789760774.520577:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.520580:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.520583:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.520586:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.520588:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.520597:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.520605:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.520627:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760774.520728:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 22319104<>22323200 00000080:00200000:1.0:1789760774.564419:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5449, 5449] 00000080:00200000:1.0:1789760774.564437:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.564444:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.564459:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22323200 i_blocks 45063 00000020:00200000:1.0:1789760774.564463:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.564473:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.564508:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22323200, count: 4096 00000080:00200000:1.0:1789760774.564516:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22323200, bytes: 4096 00000080:00200000:1.0:1789760774.564528:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.564537:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.564540:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22323200, 22327296) 0 0 00000080:00200000:1.0:1789760774.564545:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.564562:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5450, 5450] 00020000:00200000:1.0:1789760774.564648:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22323200 chunk: [22323200, 22327296) 22327296, 4096 00020000:00200000:1.0:1789760774.564654:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.564676:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5450, 5450] 00000008:00010000:1.0:1789760774.564694:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.564700:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.564712:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22323200, 22327296) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.564727:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22323200=0x154a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.564864:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.564867:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22327296 00000020:00010000:1.0:1789760774.564878:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.564882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.564885:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.564889:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.564892:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.564980:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.564989:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.565082:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760774.565190:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 22323200<>22327296 00000080:00200000:1.0:1789760774.617876:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5450, 5450] 00000080:00200000:1.0:1789760774.617902:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.617910:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.617931:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22327296 i_blocks 45063 00000020:00200000:1.0:1789760774.617936:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.617955:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.617991:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22327296, count: 4096 00000080:00200000:1.0:1789760774.618001:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22327296, bytes: 4096 00000080:00200000:1.0:1789760774.618017:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.618026:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.618028:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22327296, 22331392) 0 0 00000080:00200000:1.0:1789760774.618033:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.618050:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5451, 5451] 00020000:00200000:1.0:1789760774.618059:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22327296 chunk: [22327296, 22331392) 22331392, 4096 00020000:00200000:1.0:1789760774.618064:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.618085:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5451, 5451] 00000008:00010000:1.0:1789760774.618107:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.618113:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.618126:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22327296, 22331392) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.618748:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22327296=0x154b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.618826:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.618830:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22331392 00000020:00010000:1.0:1789760774.618840:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.618844:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.618847:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.618851:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.618854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.618864:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.618871:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.618907:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760774.619028:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 22327296<>22331392 00000080:00200000:1.0:1789760774.663211:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5451, 5451] 00000080:00200000:1.0:1789760774.663228:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.663234:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.663250:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22331392 i_blocks 45063 00000020:00200000:1.0:1789760774.663254:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.663263:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.663295:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22331392, count: 4096 00000080:00200000:1.0:1789760774.663303:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22331392, bytes: 4096 00000080:00200000:1.0:1789760774.663314:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.663323:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.663325:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22331392, 22335488) 0 0 00000080:00200000:1.0:1789760774.663330:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.663346:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5452, 5452] 00020000:00200000:1.0:1789760774.663355:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22331392 chunk: [22331392, 22335488) 22335488, 4096 00020000:00200000:1.0:1789760774.663360:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.663413:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5452, 5452] 00000008:00010000:1.0:1789760774.663434:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.663440:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.663452:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22331392, 22335488) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.663467:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22331392=0x154c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.663521:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.663523:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22335488 00000020:00010000:1.0:1789760774.663532:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.663535:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.663539:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.663542:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.663545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.663553:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.663560:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.663578:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760774.663682:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 22331392<>22335488 00000100:00100000:1.0:1789760774.663723:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760774.663748:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507333120:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760774.701867:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5452, 5452] 00000080:00200000:1.0:1789760774.701884:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.701891:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.701907:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22335488 i_blocks 45063 00000020:00200000:1.0:1789760774.701912:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.701923:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.701959:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22335488, count: 4096 00000080:00200000:1.0:1789760774.701966:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22335488, bytes: 4096 00000080:00200000:1.0:1789760774.701977:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.701987:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.701989:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22335488, 22339584) 0 0 00000080:00200000:1.0:1789760774.701994:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.702021:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5453, 5453] 00020000:00200000:1.0:1789760774.702030:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22335488 chunk: [22335488, 22339584) 22339584, 4096 00020000:00200000:1.0:1789760774.702035:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.702057:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5453, 5453] 00000008:00010000:1.0:1789760774.702085:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.702093:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.702105:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22335488, 22339584) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.702120:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22335488=0x154d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.702185:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.702188:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22339584 00000020:00010000:1.0:1789760774.702198:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.702202:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.702205:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.702209:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.702212:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.702220:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.702227:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.702244:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760774.702351:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 22335488<>22339584 00000080:00200000:1.0:1789760774.728041:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5453, 5453] 00000080:00200000:1.0:1789760774.728059:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.728066:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.728092:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22339584 i_blocks 45063 00000020:00200000:1.0:1789760774.728098:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.728108:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.728143:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22339584, count: 4096 00000080:00200000:1.0:1789760774.728150:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22339584, bytes: 4096 00000080:00200000:1.0:1789760774.728171:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.728327:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.728331:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22339584, 22343680) 0 0 00000080:00200000:1.0:1789760774.728336:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.728353:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5454, 5454] 00020000:00200000:1.0:1789760774.728362:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22339584 chunk: [22339584, 22343680) 22343680, 4096 00020000:00200000:1.0:1789760774.728367:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.728434:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5454, 5454] 00000008:00010000:1.0:1789760774.728455:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.728461:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.728476:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22339584, 22343680) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.728492:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22339584=0x154e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.728559:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.728562:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22343680 00000020:00010000:1.0:1789760774.728570:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.728574:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.728577:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.728581:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.728584:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.728592:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.728599:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.728620:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760774.728727:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 22339584<>22343680 00000080:00200000:1.0:1789760774.760076:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5454, 5454] 00000080:00200000:1.0:1789760774.760154:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.760162:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.760179:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22343680 i_blocks 45063 00000020:00200000:1.0:1789760774.760193:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.760203:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.760244:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22343680, count: 4096 00000080:00200000:1.0:1789760774.760252:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22343680, bytes: 4096 00000080:00200000:1.0:1789760774.760261:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.760270:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.760272:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22343680, 22347776) 0 0 00000080:00200000:1.0:1789760774.760277:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.760294:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5455, 5455] 00020000:00200000:1.0:1789760774.760303:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22343680 chunk: [22343680, 22347776) 22347776, 4096 00020000:00200000:1.0:1789760774.760308:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.760328:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5455, 5455] 00000008:00010000:1.0:1789760774.760347:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.760353:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.760365:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22343680, 22347776) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.760422:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22343680=0x154f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.760481:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.760485:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22347776 00000020:00010000:1.0:1789760774.760493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.760496:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.760499:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.760503:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.760506:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.760515:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.760522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.760539:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760774.760646:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22343680<>22347776 00000080:00200000:1.0:1789760774.792709:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5455, 5455] 00000080:00200000:1.0:1789760774.792726:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.792732:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.792756:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22347776 i_blocks 45063 00000020:00200000:1.0:1789760774.792761:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.792772:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.792806:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22347776, count: 4096 00000080:00200000:1.0:1789760774.792815:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22347776, bytes: 4096 00000080:00200000:1.0:1789760774.792827:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.792836:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.792838:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22347776, 22351872) 0 0 00000080:00200000:1.0:1789760774.792842:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.792858:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5456, 5456] 00020000:00200000:1.0:1789760774.792867:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22347776 chunk: [22347776, 22351872) 22351872, 4096 00020000:00200000:1.0:1789760774.792871:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.792892:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5456, 5456] 00000008:00010000:1.0:1789760774.792912:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.792919:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.792931:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22347776, 22351872) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.792955:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22347776=0x1550000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.793016:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.793020:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22351872 00000020:00010000:1.0:1789760774.793028:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.793147:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.793152:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.793163:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.793167:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.793176:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.793184:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.793204:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760774.793506:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 22347776<>22351872 00000080:00200000:1.0:1789760774.824145:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5456, 5456] 00000080:00200000:1.0:1789760774.824170:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.824177:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.824192:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22351872 i_blocks 45063 00000020:00200000:1.0:1789760774.824197:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.824207:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.824242:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22351872, count: 4096 00000080:00200000:1.0:1789760774.824249:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22351872, bytes: 4096 00000080:00200000:1.0:1789760774.824259:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.824267:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.824270:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22351872, 22355968) 0 0 00000080:00200000:1.0:1789760774.824274:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.824289:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5457, 5457] 00020000:00200000:1.0:1789760774.824297:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22351872 chunk: [22351872, 22355968) 22355968, 4096 00020000:00200000:1.0:1789760774.824301:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.824319:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5457, 5457] 00000008:00010000:1.0:1789760774.824336:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.824342:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.824354:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22351872, 22355968) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.824369:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22351872=0x1551000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.824536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.824539:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22355968 00000020:00010000:1.0:1789760774.824548:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.824551:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.824554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.824558:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.824560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.824568:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.824575:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.824594:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760774.824700:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 22351872<>22355968 00000080:00200000:1.0:1789760774.856567:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5457, 5457] 00000080:00200000:1.0:1789760774.856586:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.856592:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.856607:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22355968 i_blocks 45063 00000020:00200000:1.0:1789760774.856611:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.856622:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.856659:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22355968, count: 4096 00000080:00200000:1.0:1789760774.856667:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22355968, bytes: 4096 00000080:00200000:1.0:1789760774.856677:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.856686:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.856689:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22355968, 22360064) 0 0 00000080:00200000:1.0:1789760774.856693:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.856711:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5458, 5458] 00020000:00200000:1.0:1789760774.856720:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22355968 chunk: [22355968, 22360064) 22360064, 4096 00020000:00200000:1.0:1789760774.856724:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.856758:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5458, 5458] 00000008:00010000:1.0:1789760774.856777:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.856784:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.856795:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22355968, 22360064) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.856810:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22355968=0x1552000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.856874:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.856877:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22360064 00000020:00010000:1.0:1789760774.856887:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.856891:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.856894:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.856898:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.856901:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.856909:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.856917:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.856937:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760774.857487:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 22355968<>22360064 00000080:00200000:1.0:1789760774.883402:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5458, 5458] 00000080:00200000:1.0:1789760774.883418:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.883424:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.883439:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22360064 i_blocks 45063 00000020:00200000:1.0:1789760774.883443:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.883453:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.883489:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22360064, count: 4096 00000080:00200000:1.0:1789760774.883496:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22360064, bytes: 4096 00000080:00200000:1.0:1789760774.883507:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.883516:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.883519:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22360064, 22364160) 0 0 00000080:00200000:1.0:1789760774.883539:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.883557:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5459, 5459] 00020000:00200000:1.0:1789760774.883566:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22360064 chunk: [22360064, 22364160) 22364160, 4096 00020000:00200000:1.0:1789760774.883571:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.883589:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5459, 5459] 00000008:00010000:1.0:1789760774.883606:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.883612:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.883623:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22360064, 22364160) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.883636:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22360064=0x1553000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.883697:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.883700:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22364160 00000020:00010000:1.0:1789760774.883708:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.883712:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.883716:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.883720:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.883723:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.883732:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.883751:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.883780:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760774.884145:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 22360064<>22364160 00000080:00200000:1.0:1789760774.926220:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5459, 5459] 00000080:00200000:1.0:1789760774.926237:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.926243:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.926260:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22364160 i_blocks 45063 00000020:00200000:1.0:1789760774.926264:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.926274:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.926308:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22364160, count: 4096 00000080:00200000:1.0:1789760774.926315:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22364160, bytes: 4096 00000080:00200000:1.0:1789760774.926325:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.926335:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.926337:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22364160, 22368256) 0 0 00000080:00200000:1.0:1789760774.926342:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.926358:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5460, 5460] 00020000:00200000:1.0:1789760774.926367:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22364160 chunk: [22364160, 22368256) 22368256, 4096 00020000:00200000:1.0:1789760774.926371:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.926428:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5460, 5460] 00000008:00010000:1.0:1789760774.926447:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.926454:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.926481:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22364160, 22368256) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.926495:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22364160=0x1554000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.926562:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.926565:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22368256 00000020:00010000:1.0:1789760774.926572:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.926576:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.926579:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.926582:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.926585:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.926593:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.926600:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.926617:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760774.926904:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 22364160<>22368256 00000080:00200000:1.0:1789760774.968905:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5460, 5460] 00000080:00200000:1.0:1789760774.968986:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760774.968993:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760774.969012:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22368256 i_blocks 45063 00000020:00200000:1.0:1789760774.969017:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760774.969028:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760774.969063:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22368256, count: 4096 00000080:00200000:1.0:1789760774.969071:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22368256, bytes: 4096 00000080:00200000:1.0:1789760774.969084:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760774.969093:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760774.969096:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22368256, 22372352) 0 0 00000080:00200000:1.0:1789760774.969100:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760774.969118:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5461, 5461] 00020000:00200000:1.0:1789760774.969126:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22368256 chunk: [22368256, 22372352) 22372352, 4096 00020000:00200000:1.0:1789760774.969130:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760774.969150:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5461, 5461] 00000008:00010000:1.0:1789760774.969169:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760774.969176:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760774.969186:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22368256, 22372352) append 0 append lockpos 0 00000080:00200000:1.0:1789760774.969199:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22368256=0x1555000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760774.969321:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760774.969324:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22372352 00000020:00010000:1.0:1789760774.969535:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760774.969540:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.969545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760774.969549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760774.969552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760774.969562:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760774.969571:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760774.969598:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760774.969824:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 22368256<>22372352 00000080:00200000:1.0:1789760775.001587:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5461, 5461] 00000080:00200000:1.0:1789760775.001604:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.001610:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.001625:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22372352 i_blocks 45063 00000020:00200000:1.0:1789760775.001630:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.001640:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.001672:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22372352, count: 4096 00000080:00200000:1.0:1789760775.001680:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22372352, bytes: 4096 00000080:00200000:1.0:1789760775.001689:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.001698:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.001700:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22372352, 22376448) 0 0 00000080:00200000:1.0:1789760775.001705:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.001806:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5462, 5462] 00020000:00200000:1.0:1789760775.001815:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22372352 chunk: [22372352, 22376448) 22376448, 4096 00020000:00200000:1.0:1789760775.001820:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.001839:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5462, 5462] 00000008:00010000:1.0:1789760775.001858:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.001865:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.001877:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22372352, 22376448) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.001894:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22372352=0x1556000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.001953:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.001956:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22376448 00000020:00010000:1.0:1789760775.001965:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.001969:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.001972:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.001975:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.001978:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.001988:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.001995:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760775.002012:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760775.002118:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 22372352<>22376448 00000080:00200000:1.0:1789760775.028568:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5462, 5462] 00000080:00200000:1.0:1789760775.028589:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.028595:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.028614:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22376448 i_blocks 45063 00000020:00200000:1.0:1789760775.028619:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.028633:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.028669:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22376448, count: 4096 00000080:00200000:1.0:1789760775.028677:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22376448, bytes: 4096 00000080:00200000:1.0:1789760775.028692:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.028701:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.028704:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22376448, 22380544) 0 0 00000080:00200000:1.0:1789760775.028709:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.029483:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5463, 5463] 00020000:00200000:1.0:1789760775.029494:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22376448 chunk: [22376448, 22380544) 22380544, 4096 00020000:00200000:1.0:1789760775.029498:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.029520:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5463, 5463] 00000008:00010000:1.0:1789760775.029539:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.029545:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.029557:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22376448, 22380544) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.029572:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22376448=0x1557000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.029638:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.029641:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22380544 00000020:00010000:1.0:1789760775.029648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.029652:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.029655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.029659:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.029662:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.029670:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.029677:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760775.029703:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760775.029906:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 22376448<>22380544 00000080:00200000:1.0:1789760775.066282:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5463, 5463] 00000080:00200000:1.0:1789760775.066297:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.066303:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.066319:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22380544 i_blocks 45063 00000020:00200000:1.0:1789760775.066323:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.066430:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.066469:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22380544, count: 4096 00000080:00200000:1.0:1789760775.066476:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22380544, bytes: 4096 00000080:00200000:1.0:1789760775.066486:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.066557:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.066560:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22380544, 22384640) 0 0 00000080:00200000:1.0:1789760775.066564:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.066582:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5464, 5464] 00020000:00200000:1.0:1789760775.066591:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22380544 chunk: [22380544, 22384640) 22384640, 4096 00020000:00200000:1.0:1789760775.066596:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.066617:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5464, 5464] 00000008:00010000:1.0:1789760775.066635:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.066642:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.066654:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22380544, 22384640) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.066667:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22380544=0x1558000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.066726:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.066728:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22384640 00000020:00010000:1.0:1789760775.066748:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.066752:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.066755:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.066758:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.066760:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.066768:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.066775:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760775.066793:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760775.066894:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 22380544<>22384640 00000080:00200000:2.0:1789760775.104332:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5464, 5464] 00000080:00200000:2.0:1789760775.104359:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.104368:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.106258:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22384640 i_blocks 45063 00000020:00200000:2.0:1789760775.106272:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.106293:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.106331:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22384640, count: 4096 00000080:00200000:2.0:1789760775.106341:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22384640, bytes: 4096 00000080:00200000:2.0:1789760775.106355:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.106364:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.106366:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22384640, 22388736) 0 0 00000080:00200000:2.0:1789760775.106372:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.106405:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5465, 5465] 00020000:00200000:2.0:1789760775.106414:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22384640 chunk: [22384640, 22388736) 22388736, 4096 00020000:00200000:2.0:1789760775.106419:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.106442:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5465, 5465] 00000008:00010000:2.0:1789760775.106461:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.106467:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.106480:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22384640, 22388736) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.107639:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22384640=0x1559000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.107730:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.107743:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22388736 00000020:00010000:2.0:1789760775.107753:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.107757:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.107761:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.107765:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.107768:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.107779:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.107787:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.107817:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760775.107918:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022d80 - object 0x280000401:3 offset 22384640<>22388736 00000080:00200000:2.0:1789760775.161907:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5465, 5465] 00000080:00200000:2.0:1789760775.161928:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.161935:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.161952:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22388736 i_blocks 45063 00000020:00200000:2.0:1789760775.161956:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.161968:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.162003:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22388736, count: 4096 00000080:00200000:2.0:1789760775.162011:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22388736, bytes: 4096 00000080:00200000:2.0:1789760775.162023:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.162042:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.162045:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22388736, 22392832) 0 0 00000080:00200000:2.0:1789760775.162049:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.162067:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5466, 5466] 00020000:00200000:2.0:1789760775.162076:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22388736 chunk: [22388736, 22392832) 22392832, 4096 00020000:00200000:2.0:1789760775.162081:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.162102:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5466, 5466] 00000008:00010000:2.0:1789760775.162121:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.162127:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.162139:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22388736, 22392832) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.162153:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22388736=0x155a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.162215:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.162218:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22392832 00000020:00010000:2.0:1789760775.162227:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.162231:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.162234:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.162238:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.162242:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.162250:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.162257:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.162276:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760775.162434:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 22388736<>22392832 00000100:00100000:1.0:1789760775.205795:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507335040:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760775.270168:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5467, 5467] 00000080:00200000:1.0:1789760775.270184:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.270189:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.270206:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22396928 i_blocks 45063 00000020:00200000:1.0:1789760775.270210:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.270220:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.270254:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22396928, count: 4096 00000080:00200000:1.0:1789760775.270261:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22396928, bytes: 4096 00000080:00200000:1.0:1789760775.270272:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.270281:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.270283:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22396928, 22401024) 0 0 00000080:00200000:1.0:1789760775.270288:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.270304:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5468, 5468] 00020000:00200000:1.0:1789760775.270312:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22396928 chunk: [22396928, 22401024) 22401024, 4096 00020000:00200000:1.0:1789760775.270317:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.270336:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5468, 5468] 00000008:00010000:1.0:1789760775.270355:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.270360:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.270372:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22396928, 22401024) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.270410:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22396928=0x155c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.270470:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.270473:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22401024 00000020:00010000:1.0:1789760775.270482:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.270486:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.270489:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.270492:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.270495:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.270503:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.270510:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760775.270631:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760775.270745:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 22396928<>22401024 00000080:00200000:1.0:1789760775.304271:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5468, 5468] 00000080:00200000:1.0:1789760775.304286:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.304292:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.304304:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22401024 i_blocks 45063 00000020:00200000:1.0:1789760775.304308:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.304315:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.304338:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22401024, count: 4096 00000080:00200000:1.0:1789760775.304345:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22401024, bytes: 4096 00000080:00200000:1.0:1789760775.304353:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.304360:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.304362:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22401024, 22405120) 0 0 00000080:00200000:1.0:1789760775.304366:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.304410:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5469, 5469] 00020000:00200000:1.0:1789760775.304417:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22401024 chunk: [22401024, 22405120) 22405120, 4096 00020000:00200000:1.0:1789760775.304420:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.304435:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5469, 5469] 00000008:00010000:1.0:1789760775.304448:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.304452:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.304461:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22401024, 22405120) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.304471:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22401024=0x155d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.304507:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.304510:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22405120 00000020:00010000:1.0:1789760775.304516:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.304519:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.304522:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.304525:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.304528:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.304534:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.304540:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.349720:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5469, 5469] 00000080:00200000:1.0:1789760775.349742:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.349747:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.349759:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22405120 i_blocks 45063 00000020:00200000:1.0:1789760775.349763:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.349771:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.349795:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22405120, count: 4096 00000080:00200000:1.0:1789760775.349800:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22405120, bytes: 4096 00000080:00200000:1.0:1789760775.349807:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.349814:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.349816:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22405120, 22409216) 0 0 00000080:00200000:1.0:1789760775.349819:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.349831:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5470, 5470] 00020000:00200000:1.0:1789760775.349836:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22405120 chunk: [22405120, 22409216) 22409216, 4096 00020000:00200000:1.0:1789760775.349840:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.349853:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5470, 5470] 00000008:00010000:1.0:1789760775.349865:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.349868:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.349876:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22405120, 22409216) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.349886:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22405120=0x155e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.349926:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.349928:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22409216 00000020:00010000:1.0:1789760775.349934:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.349937:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.349940:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.349943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.349946:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.349952:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.349963:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760775.349977:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760775.350052:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 22405120<>22409216 00000080:00200000:1.0:1789760775.424462:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5470, 5470] 00000080:00200000:1.0:1789760775.424482:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.424488:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.424505:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22409216 i_blocks 45063 00000020:00200000:1.0:1789760775.424509:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.424520:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.424556:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22409216, count: 4096 00000080:00200000:1.0:1789760775.424563:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22409216, bytes: 4096 00000080:00200000:1.0:1789760775.424576:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.424588:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.424590:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22409216, 22413312) 0 0 00000080:00200000:1.0:1789760775.424595:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.424611:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5471, 5471] 00020000:00200000:1.0:1789760775.424620:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22409216 chunk: [22409216, 22413312) 22413312, 4096 00020000:00200000:1.0:1789760775.424625:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.424645:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5471, 5471] 00000008:00010000:1.0:1789760775.424666:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.424672:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.424684:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22409216, 22413312) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.426020:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22409216=0x155f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.426120:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.426123:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22413312 00000020:00010000:1.0:1789760775.426134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.426138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.426142:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.426145:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.426148:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.426159:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.426183:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760775.426216:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760775.426331:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 22409216<>22413312 00000080:00200000:1.0:1789760775.472961:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5471, 5471] 00000080:00200000:1.0:1789760775.472978:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760775.472984:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760775.473000:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22413312 i_blocks 45063 00000020:00200000:1.0:1789760775.473004:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760775.473015:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760775.473052:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22413312, count: 4096 00000080:00200000:1.0:1789760775.473060:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22413312, bytes: 4096 00000080:00200000:1.0:1789760775.473073:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760775.473082:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760775.473084:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22413312, 22417408) 0 0 00000080:00200000:1.0:1789760775.473090:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760775.473108:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5472, 5472] 00020000:00200000:1.0:1789760775.473718:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22413312 chunk: [22413312, 22417408) 22417408, 4096 00020000:00200000:1.0:1789760775.473725:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760775.473755:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5472, 5472] 00000008:00010000:1.0:1789760775.473775:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760775.473781:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760775.473793:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22413312, 22417408) append 0 append lockpos 0 00000080:00200000:1.0:1789760775.473808:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22413312=0x1560000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760775.473872:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760775.473875:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22417408 00000020:00010000:1.0:1789760775.473883:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760775.473887:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.473890:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760775.473894:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760775.473897:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760775.473904:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760775.473911:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760775.473932:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760775.474035:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22413312<>22417408 00000080:00200000:2.0:1789760775.518159:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5472, 5472] 00000080:00200000:2.0:1789760775.518176:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.518183:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.518199:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22417408 i_blocks 45063 00000020:00200000:2.0:1789760775.518203:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.518214:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.518249:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22417408, count: 4096 00000080:00200000:2.0:1789760775.518257:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22417408, bytes: 4096 00000080:00200000:2.0:1789760775.518268:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.518278:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.518280:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22417408, 22421504) 0 0 00000080:00200000:2.0:1789760775.518285:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.518303:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5473, 5473] 00020000:00200000:2.0:1789760775.518313:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22417408 chunk: [22417408, 22421504) 22421504, 4096 00020000:00200000:2.0:1789760775.518318:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.518339:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5473, 5473] 00000008:00010000:2.0:1789760775.518357:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.518362:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.518374:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22417408, 22421504) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.518414:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22417408=0x1561000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.518470:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.518473:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22421504 00000020:00010000:2.0:1789760775.518480:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.518484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.518487:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.518490:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.518493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.518501:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.518508:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.518523:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760775.518703:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 22417408<>22421504 00000080:00200000:2.0:1789760775.598130:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5474, 5474] 00000080:00200000:2.0:1789760775.598149:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.598155:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.598171:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22425600 i_blocks 45063 00000020:00200000:2.0:1789760775.598185:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.598196:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.598231:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22425600, count: 4096 00000080:00200000:2.0:1789760775.598239:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22425600, bytes: 4096 00000080:00200000:2.0:1789760775.598251:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.598260:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.598263:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22425600, 22429696) 0 0 00000080:00200000:2.0:1789760775.598267:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.598283:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5475, 5475] 00020000:00200000:2.0:1789760775.598292:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22425600 chunk: [22425600, 22429696) 22429696, 4096 00020000:00200000:2.0:1789760775.598296:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.598316:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5475, 5475] 00000008:00010000:2.0:1789760775.598334:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.598340:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.598351:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22425600, 22429696) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.598365:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22425600=0x1563000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.598462:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.598465:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22429696 00000020:00010000:2.0:1789760775.598473:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.598477:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.598480:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.598484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.598487:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.598495:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.598502:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.598523:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760775.598626:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 22425600<>22429696 00000080:00200000:2.0:1789760775.632968:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5475, 5475] 00000080:00200000:2.0:1789760775.632990:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.632997:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.633010:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22429696 i_blocks 45063 00000020:00200000:2.0:1789760775.633014:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.633026:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.633055:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22429696, count: 4096 00000080:00200000:2.0:1789760775.633062:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22429696, bytes: 4096 00000080:00200000:2.0:1789760775.633072:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.633088:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.633090:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22429696, 22433792) 0 0 00000080:00200000:2.0:1789760775.633093:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.633119:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5476, 5476] 00020000:00200000:2.0:1789760775.633127:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22429696 chunk: [22429696, 22433792) 22433792, 4096 00020000:00200000:2.0:1789760775.633131:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.633146:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5476, 5476] 00000008:00010000:2.0:1789760775.633160:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.633165:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.633173:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22429696, 22433792) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.633183:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22429696=0x1564000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.633244:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.633247:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22433792 00000020:00010000:2.0:1789760775.633255:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.633258:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.633261:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.633264:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.633266:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.633274:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.633280:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.633299:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760775.633427:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 22429696<>22433792 00000080:00200000:2.0:1789760775.675175:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5476, 5476] 00000080:00200000:2.0:1789760775.675193:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.675200:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.675225:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22433792 i_blocks 45063 00000020:00200000:2.0:1789760775.675230:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.675241:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.675276:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22433792, count: 4096 00000080:00200000:2.0:1789760775.675284:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22433792, bytes: 4096 00000080:00200000:2.0:1789760775.675296:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.675308:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.675310:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22433792, 22437888) 0 0 00000080:00200000:2.0:1789760775.675315:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.675331:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5477, 5477] 00020000:00200000:2.0:1789760775.675339:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22433792 chunk: [22433792, 22437888) 22437888, 4096 00020000:00200000:2.0:1789760775.675343:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.675362:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5477, 5477] 00000008:00010000:2.0:1789760775.675408:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.675415:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.675426:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22433792, 22437888) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.675439:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22433792=0x1565000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.675506:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.675509:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22437888 00000020:00010000:2.0:1789760775.675518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.675522:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.675525:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.675529:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.675532:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.675539:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.675545:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.675561:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760775.675677:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 22433792<>22437888 00000080:00200000:2.0:1789760775.718642:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5477, 5477] 00000080:00200000:2.0:1789760775.718659:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.718667:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.718683:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22437888 i_blocks 45063 00000020:00200000:2.0:1789760775.718686:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.718696:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.718784:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22437888, count: 4096 00000080:00200000:2.0:1789760775.718792:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22437888, bytes: 4096 00000080:00200000:2.0:1789760775.718803:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.718812:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.718814:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22437888, 22441984) 0 0 00000080:00200000:2.0:1789760775.718819:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.718834:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5478, 5478] 00020000:00200000:2.0:1789760775.718842:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22437888 chunk: [22437888, 22441984) 22441984, 4096 00020000:00200000:2.0:1789760775.718847:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.718867:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5478, 5478] 00000008:00010000:2.0:1789760775.718886:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.718892:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.718903:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22437888, 22441984) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.718918:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22437888=0x1566000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.718975:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.718979:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22441984 00000020:00010000:2.0:1789760775.718987:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.718991:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.718995:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.718998:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.719001:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.719008:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.719015:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.719032:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760775.719152:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 22437888<>22441984 00000080:00200000:2.0:1789760775.768915:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5478, 5478] 00000080:00200000:2.0:1789760775.768934:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.768941:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.768959:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22441984 i_blocks 45063 00000020:00200000:2.0:1789760775.768963:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.768973:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.769012:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22441984, count: 4096 00000080:00200000:2.0:1789760775.769019:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22441984, bytes: 4096 00000080:00200000:2.0:1789760775.769032:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.769042:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.769044:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22441984, 22446080) 0 0 00000080:00200000:2.0:1789760775.769049:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.769066:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5479, 5479] 00020000:00200000:2.0:1789760775.769424:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22441984 chunk: [22441984, 22446080) 22446080, 4096 00020000:00200000:2.0:1789760775.769431:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.769453:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5479, 5479] 00000008:00010000:2.0:1789760775.769471:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.769477:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.769489:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22441984, 22446080) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.769504:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22441984=0x1567000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.769565:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.769568:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22446080 00000020:00010000:2.0:1789760775.769577:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.769580:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.769583:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.769586:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.769589:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.769598:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.769604:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.769626:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760775.769740:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 22441984<>22446080 00000080:00200000:2.0:1789760775.826580:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5479, 5479] 00000080:00200000:2.0:1789760775.826599:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.826606:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.826623:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22446080 i_blocks 45063 00000020:00200000:2.0:1789760775.826627:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.826637:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.826687:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22446080, count: 4096 00000080:00200000:2.0:1789760775.826694:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22446080, bytes: 4096 00000080:00200000:2.0:1789760775.826707:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.826717:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.826719:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22446080, 22450176) 0 0 00000080:00200000:2.0:1789760775.826724:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.826754:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5480, 5480] 00020000:00200000:2.0:1789760775.826763:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22446080 chunk: [22446080, 22450176) 22450176, 4096 00020000:00200000:2.0:1789760775.826768:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.826790:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5480, 5480] 00000008:00010000:2.0:1789760775.826809:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.826816:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.826828:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22446080, 22450176) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.826843:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22446080=0x1568000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.826912:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.826915:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22450176 00000020:00010000:2.0:1789760775.826923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.826927:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.826930:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.826934:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.826937:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.826946:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.826953:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.826969:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760775.827368:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 22446080<>22450176 00000080:00200000:2.0:1789760775.892456:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5480, 5480] 00000080:00200000:2.0:1789760775.892475:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.892482:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.892501:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22450176 i_blocks 45063 00000020:00200000:2.0:1789760775.892505:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.892517:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.892554:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22450176, count: 4096 00000080:00200000:2.0:1789760775.892562:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22450176, bytes: 4096 00000080:00200000:2.0:1789760775.892572:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.892581:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.892584:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22450176, 22454272) 0 0 00000080:00200000:2.0:1789760775.892588:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.892608:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5481, 5481] 00020000:00200000:2.0:1789760775.892617:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22450176 chunk: [22450176, 22454272) 22454272, 4096 00020000:00200000:2.0:1789760775.892621:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.892642:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5481, 5481] 00000008:00010000:2.0:1789760775.892661:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.892668:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.892681:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22450176, 22454272) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.892696:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22450176=0x1569000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.892769:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.892772:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22454272 00000020:00010000:2.0:1789760775.892780:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.892784:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.892787:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.892790:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.892793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.892801:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.892808:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.892827:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760775.892939:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 22450176<>22454272 00000080:00200000:2.0:1789760775.951220:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5481, 5481] 00000080:00200000:2.0:1789760775.951237:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760775.951244:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760775.951262:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22454272 i_blocks 45063 00000020:00200000:2.0:1789760775.951267:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760775.951278:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760775.951313:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22454272, count: 4096 00000080:00200000:2.0:1789760775.951319:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22454272, bytes: 4096 00000080:00200000:2.0:1789760775.951653:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760775.951664:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760775.951666:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22454272, 22458368) 0 0 00000080:00200000:2.0:1789760775.951671:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760775.951688:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5482, 5482] 00020000:00200000:2.0:1789760775.951696:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22454272 chunk: [22454272, 22458368) 22458368, 4096 00020000:00200000:2.0:1789760775.951700:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760775.951723:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5482, 5482] 00000008:00010000:2.0:1789760775.951753:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760775.951759:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760775.951771:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22454272, 22458368) append 0 append lockpos 0 00000080:00200000:2.0:1789760775.951784:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22454272=0x156a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760775.951840:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760775.951843:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22458368 00000020:00010000:2.0:1789760775.951851:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760775.951854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.951857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760775.951861:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760775.951865:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760775.951873:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760775.951880:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760775.951899:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760775.951999:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 22454272<>22458368 00000100:00100000:1.0:1789760775.997268:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760775.997305:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507337088:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760776.041066:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5483, 5483] 00000080:00200000:1.0:1789760776.041101:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.041109:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.041131:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22462464 i_blocks 45063 00000020:00200000:1.0:1789760776.041136:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.041158:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.041199:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22462464, count: 4096 00000080:00200000:1.0:1789760776.041209:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22462464, bytes: 4096 00000080:00200000:1.0:1789760776.041224:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.041233:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.041236:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22462464, 22466560) 0 0 00000080:00200000:1.0:1789760776.041240:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.041258:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5484, 5484] 00020000:00200000:1.0:1789760776.041266:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22462464 chunk: [22462464, 22466560) 22466560, 4096 00020000:00200000:1.0:1789760776.041271:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.041294:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5484, 5484] 00000008:00010000:1.0:1789760776.041314:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.041320:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.041344:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22462464, 22466560) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.041362:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22462464=0x156c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.041581:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.041585:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22466560 00000020:00010000:1.0:1789760776.041594:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.041597:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.041601:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.041604:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.041607:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.041615:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.041622:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.041664:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760776.041790:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 22462464<>22466560 00000080:00200000:1.0:1789760776.069548:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5484, 5484] 00000080:00200000:1.0:1789760776.069567:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.069573:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.069589:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22466560 i_blocks 45063 00000020:00200000:1.0:1789760776.069593:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.069603:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.069639:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22466560, count: 4096 00000080:00200000:1.0:1789760776.069647:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22466560, bytes: 4096 00000080:00200000:1.0:1789760776.069659:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.069668:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.069670:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22466560, 22470656) 0 0 00000080:00200000:1.0:1789760776.069675:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.069692:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5485, 5485] 00020000:00200000:1.0:1789760776.069702:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22466560 chunk: [22466560, 22470656) 22470656, 4096 00020000:00200000:1.0:1789760776.069706:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.069735:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5485, 5485] 00000008:00010000:1.0:1789760776.069753:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.069759:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.069770:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22466560, 22470656) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.069784:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22466560=0x156d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.069833:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.069836:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22470656 00000020:00010000:1.0:1789760776.069844:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.069848:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.069850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.069854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.069856:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.069864:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.069870:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.107875:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5485, 5485] 00000080:00200000:1.0:1789760776.107891:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.107898:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.107914:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22470656 i_blocks 45063 00000020:00200000:1.0:1789760776.107917:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.107928:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.107962:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22470656, count: 4096 00000080:00200000:1.0:1789760776.107969:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22470656, bytes: 4096 00000080:00200000:1.0:1789760776.107985:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.107994:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.107996:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22470656, 22474752) 0 0 00000080:00200000:1.0:1789760776.108000:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.108019:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5486, 5486] 00020000:00200000:1.0:1789760776.108028:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22470656 chunk: [22470656, 22474752) 22474752, 4096 00020000:00200000:1.0:1789760776.108033:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.108054:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5486, 5486] 00000008:00010000:1.0:1789760776.108073:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.108079:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.108090:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22470656, 22474752) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.108105:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22470656=0x156e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.108168:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.108171:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22474752 00000020:00010000:1.0:1789760776.108181:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.108185:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.108188:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.108192:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.108195:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.108204:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.108210:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.108229:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760776.108330:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 22470656<>22474752 00000100:00100000:1.0:1789760776.108440:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507337472:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760776.173111:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5486, 5486] 00000080:00200000:1.0:1789760776.173132:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.173139:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.173158:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22474752 i_blocks 45063 00000020:00200000:1.0:1789760776.173162:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.173176:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.173217:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22474752, count: 4096 00000080:00200000:1.0:1789760776.173225:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22474752, bytes: 4096 00000080:00200000:1.0:1789760776.173240:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.173249:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.173251:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22474752, 22478848) 0 0 00000080:00200000:1.0:1789760776.173256:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.173273:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5487, 5487] 00020000:00200000:1.0:1789760776.173282:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22474752 chunk: [22474752, 22478848) 22478848, 4096 00020000:00200000:1.0:1789760776.173286:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.173308:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5487, 5487] 00000008:00010000:1.0:1789760776.173328:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.173334:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.173345:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22474752, 22478848) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.173359:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22474752=0x156f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.173452:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.173456:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22478848 00000020:00010000:1.0:1789760776.173465:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.173468:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.173472:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.173475:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.173479:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.173488:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.173496:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.173516:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760776.173633:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 22474752<>22478848 00000080:00200000:1.0:1789760776.223673:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5487, 5487] 00000080:00200000:1.0:1789760776.223690:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.223696:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.223714:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22478848 i_blocks 45063 00000020:00200000:1.0:1789760776.223717:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.223737:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.223780:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22478848, count: 4096 00000080:00200000:1.0:1789760776.223787:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22478848, bytes: 4096 00000080:00200000:1.0:1789760776.223798:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.223806:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.223808:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22478848, 22482944) 0 0 00000080:00200000:1.0:1789760776.223813:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.223828:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5488, 5488] 00020000:00200000:1.0:1789760776.223836:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22478848 chunk: [22478848, 22482944) 22482944, 4096 00020000:00200000:1.0:1789760776.223841:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.223862:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5488, 5488] 00000008:00010000:1.0:1789760776.223879:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.223885:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.223898:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22478848, 22482944) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.223912:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22478848=0x1570000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.223970:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.223973:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22482944 00000020:00010000:1.0:1789760776.223983:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.223986:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.223989:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.223993:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.223996:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.224004:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.224011:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.224028:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760776.226916:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 22478848<>22482944 00000080:00200000:1.0:1789760776.255008:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5488, 5488] 00000080:00200000:1.0:1789760776.255142:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.255149:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.255163:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22482944 i_blocks 45063 00000020:00200000:1.0:1789760776.255166:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.255175:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.255211:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22482944, count: 4096 00000080:00200000:1.0:1789760776.255219:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22482944, bytes: 4096 00000080:00200000:1.0:1789760776.255229:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.255237:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.255240:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22482944, 22487040) 0 0 00000080:00200000:1.0:1789760776.255244:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.255263:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5489, 5489] 00020000:00200000:1.0:1789760776.255272:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22482944 chunk: [22482944, 22487040) 22487040, 4096 00020000:00200000:1.0:1789760776.255276:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.255297:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5489, 5489] 00000008:00010000:1.0:1789760776.255314:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.255320:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.255332:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22482944, 22487040) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.255347:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22482944=0x1571000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.255496:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.255498:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22487040 00000020:00010000:1.0:1789760776.255507:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.255510:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.255514:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.255517:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.255520:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.255528:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.255535:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.255552:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760776.255978:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 22482944<>22487040 00000080:00200000:1.0:1789760776.291111:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5489, 5489] 00000080:00200000:1.0:1789760776.291128:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.291134:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.291150:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22487040 i_blocks 45063 00000020:00200000:1.0:1789760776.291160:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.291232:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.291270:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22487040, count: 4096 00000080:00200000:1.0:1789760776.291277:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22487040, bytes: 4096 00000080:00200000:1.0:1789760776.291294:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.291302:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.291305:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22487040, 22491136) 0 0 00000080:00200000:1.0:1789760776.291309:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.291325:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5490, 5490] 00020000:00200000:1.0:1789760776.291333:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22487040 chunk: [22487040, 22491136) 22491136, 4096 00020000:00200000:1.0:1789760776.291337:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.291358:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5490, 5490] 00000008:00010000:1.0:1789760776.291415:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.291421:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.291434:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22487040, 22491136) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.291449:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22487040=0x1572000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.291507:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.291510:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22491136 00000020:00010000:1.0:1789760776.291518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.291522:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.291524:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.291528:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.291531:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.291539:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.291546:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.291564:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760776.291670:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 22487040<>22491136 00000080:00200000:1.0:1789760776.322432:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5490, 5490] 00000080:00200000:1.0:1789760776.322450:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.322456:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.322471:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22491136 i_blocks 45063 00000020:00200000:1.0:1789760776.322475:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.322485:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.322519:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22491136, count: 4096 00000080:00200000:1.0:1789760776.322526:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22491136, bytes: 4096 00000080:00200000:1.0:1789760776.322537:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.322546:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.322548:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22491136, 22495232) 0 0 00000080:00200000:1.0:1789760776.322552:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.322568:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5491, 5491] 00020000:00200000:1.0:1789760776.322575:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22491136 chunk: [22491136, 22495232) 22495232, 4096 00020000:00200000:1.0:1789760776.322580:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.322598:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5491, 5491] 00000008:00010000:1.0:1789760776.322617:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.322623:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.322634:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22491136, 22495232) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.322648:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22491136=0x1573000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.322706:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.322708:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22495232 00000020:00010000:1.0:1789760776.322717:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.322720:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.322736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.322741:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.322743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.322752:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.322837:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.322857:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760776.323049:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 22491136<>22495232 00000080:00200000:2.0:1789760776.368121:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5491, 5491] 00000080:00200000:2.0:1789760776.368138:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.368145:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.368161:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22495232 i_blocks 45063 00000020:00200000:2.0:1789760776.368165:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.368185:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.368218:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22495232, count: 4096 00000080:00200000:2.0:1789760776.368226:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22495232, bytes: 4096 00000080:00200000:2.0:1789760776.368236:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.368245:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.368248:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22495232, 22499328) 0 0 00000080:00200000:2.0:1789760776.368252:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.368268:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5492, 5492] 00020000:00200000:2.0:1789760776.368277:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22495232 chunk: [22495232, 22499328) 22499328, 4096 00020000:00200000:2.0:1789760776.368282:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.368300:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5492, 5492] 00000008:00010000:2.0:1789760776.368317:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.368322:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.368334:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22495232, 22499328) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.368348:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22495232=0x1574000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.368429:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.368432:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22499328 00000020:00010000:2.0:1789760776.368440:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.368443:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.368446:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.368450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.368453:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.368459:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.368466:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.368480:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760776.368576:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 22495232<>22499328 00000080:00200000:2.0:1789760776.406192:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5492, 5492] 00000080:00200000:2.0:1789760776.406209:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.406216:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.406232:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22499328 i_blocks 45063 00000020:00200000:2.0:1789760776.406236:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.406246:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.406283:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22499328, count: 4096 00000080:00200000:2.0:1789760776.406290:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22499328, bytes: 4096 00000080:00200000:2.0:1789760776.406300:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.406308:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.406310:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22499328, 22503424) 0 0 00000080:00200000:2.0:1789760776.406315:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.406332:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5493, 5493] 00020000:00200000:2.0:1789760776.406340:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22499328 chunk: [22499328, 22503424) 22503424, 4096 00020000:00200000:2.0:1789760776.406345:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.406523:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5493, 5493] 00000008:00010000:2.0:1789760776.406543:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.406550:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.406562:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22499328, 22503424) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.406576:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22499328=0x1575000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.406705:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.406708:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22503424 00000020:00010000:2.0:1789760776.406716:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.406720:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.406733:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.406737:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.406740:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.406748:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.406755:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.406773:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760776.406873:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 22499328<>22503424 00000080:00200000:2.0:1789760776.438627:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5493, 5493] 00000080:00200000:2.0:1789760776.438660:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.438668:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.438685:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22503424 i_blocks 45063 00000020:00200000:2.0:1789760776.438689:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.438700:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.438745:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22503424, count: 4096 00000080:00200000:2.0:1789760776.438752:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22503424, bytes: 4096 00000080:00200000:2.0:1789760776.438763:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.438773:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.438775:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22503424, 22507520) 0 0 00000080:00200000:2.0:1789760776.438779:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.438797:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5494, 5494] 00020000:00200000:2.0:1789760776.438805:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22503424 chunk: [22503424, 22507520) 22507520, 4096 00020000:00200000:2.0:1789760776.438810:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.438831:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5494, 5494] 00000008:00010000:2.0:1789760776.438849:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.438855:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.438868:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22503424, 22507520) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.438882:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22503424=0x1576000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.438941:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.438945:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22507520 00000020:00010000:2.0:1789760776.438953:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.438957:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.438960:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.438964:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.438967:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.438980:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.438988:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.439007:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760776.439134:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 22503424<>22507520 00000080:00200000:2.0:1789760776.476205:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5494, 5494] 00000080:00200000:2.0:1789760776.476225:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.476233:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.476251:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22507520 i_blocks 45063 00000020:00200000:2.0:1789760776.476255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.476266:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.476503:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22507520, count: 4096 00000080:00200000:2.0:1789760776.476512:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22507520, bytes: 4096 00000080:00200000:2.0:1789760776.476522:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.476531:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.476534:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22507520, 22511616) 0 0 00000080:00200000:2.0:1789760776.476539:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.476556:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5495, 5495] 00020000:00200000:2.0:1789760776.476564:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22507520 chunk: [22507520, 22511616) 22511616, 4096 00020000:00200000:2.0:1789760776.476569:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.476591:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5495, 5495] 00000008:00010000:2.0:1789760776.476611:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.476618:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.476629:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22507520, 22511616) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.476833:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22507520=0x1577000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.476898:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.476902:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22511616 00000020:00010000:2.0:1789760776.476910:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.476914:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.476917:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.476926:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.476929:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.476936:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.476944:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.476963:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760776.477053:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 22507520<>22511616 00000080:00200000:2.0:1789760776.513058:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5495, 5495] 00000080:00200000:2.0:1789760776.513077:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.513084:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.513101:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22511616 i_blocks 45063 00000020:00200000:2.0:1789760776.513105:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.513116:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.513155:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22511616, count: 4096 00000080:00200000:2.0:1789760776.513163:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22511616, bytes: 4096 00000080:00200000:2.0:1789760776.513411:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.513422:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.513424:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22511616, 22515712) 0 0 00000080:00200000:2.0:1789760776.513429:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.513445:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5496, 5496] 00020000:00200000:2.0:1789760776.513454:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22511616 chunk: [22511616, 22515712) 22515712, 4096 00020000:00200000:2.0:1789760776.513458:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.513480:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5496, 5496] 00000008:00010000:2.0:1789760776.513498:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.513660:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.513675:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22511616, 22515712) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.513690:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22511616=0x1578000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.513860:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.513864:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22515712 00000020:00010000:2.0:1789760776.513873:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.513877:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.513880:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.513884:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.513887:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.513897:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.513904:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.513934:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760776.514280:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 22511616<>22515712 00000080:00200000:2.0:1789760776.561807:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5496, 5496] 00000080:00200000:2.0:1789760776.561834:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.561842:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.561859:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22515712 i_blocks 45063 00000020:00200000:2.0:1789760776.561863:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.561873:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.561911:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22515712, count: 4096 00000080:00200000:2.0:1789760776.561918:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22515712, bytes: 4096 00000080:00200000:2.0:1789760776.561934:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.561943:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.561946:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22515712, 22519808) 0 0 00000080:00200000:2.0:1789760776.561950:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.561968:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5497, 5497] 00020000:00200000:2.0:1789760776.561978:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22515712 chunk: [22515712, 22519808) 22519808, 4096 00020000:00200000:2.0:1789760776.561983:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.562005:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5497, 5497] 00000008:00010000:2.0:1789760776.562025:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.562031:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.562045:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22515712, 22519808) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.562061:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22515712=0x1579000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.562124:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.562128:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22519808 00000020:00010000:2.0:1789760776.562138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.562142:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.562144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.562148:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.562151:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.562159:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.562179:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.562197:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760776.562314:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 22515712<>22519808 00000080:00200000:2.0:1789760776.599255:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5497, 5497] 00000080:00200000:2.0:1789760776.599272:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.599350:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.599367:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22519808 i_blocks 45063 00000020:00200000:2.0:1789760776.599372:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.599407:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.599444:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22519808, count: 4096 00000080:00200000:2.0:1789760776.599451:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22519808, bytes: 4096 00000080:00200000:2.0:1789760776.599462:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.599471:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.599473:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22519808, 22523904) 0 0 00000080:00200000:2.0:1789760776.599477:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.599494:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5498, 5498] 00020000:00200000:2.0:1789760776.599502:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22519808 chunk: [22519808, 22523904) 22523904, 4096 00020000:00200000:2.0:1789760776.599507:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.599526:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5498, 5498] 00000008:00010000:2.0:1789760776.599544:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.599550:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.599561:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22519808, 22523904) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.599576:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22519808=0x157a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.599634:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.599637:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22523904 00000020:00010000:2.0:1789760776.599645:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.599649:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.599652:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.599655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.599658:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.599668:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.599676:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.599888:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760776.600062:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 22519808<>22523904 00000100:00100000:1.0:1789760776.626018:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507339136:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760776.664634:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5499, 5499] 00000080:00200000:2.0:1789760776.664650:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.664657:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.664673:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22528000 i_blocks 45063 00000020:00200000:2.0:1789760776.664677:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.664687:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.664732:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22528000, count: 4096 00000080:00200000:2.0:1789760776.664741:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22528000, bytes: 4096 00000080:00200000:2.0:1789760776.664752:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.664913:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.664916:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22528000, 22532096) 0 0 00000080:00200000:2.0:1789760776.664921:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.664938:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5500, 5500] 00020000:00200000:2.0:1789760776.664948:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22528000 chunk: [22528000, 22532096) 22532096, 4096 00020000:00200000:2.0:1789760776.664952:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.665010:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5500, 5500] 00000008:00010000:2.0:1789760776.665029:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.665035:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.665047:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22528000, 22532096) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.665061:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22528000=0x157c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.665126:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.665129:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22532096 00000020:00010000:2.0:1789760776.665138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.665142:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.665145:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.665148:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.665152:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.665159:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.665166:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.665184:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760776.665307:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 22528000<>22532096 00000080:00200000:2.0:1789760776.705460:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5500, 5500] 00000080:00200000:2.0:1789760776.705490:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760776.705498:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760776.705517:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22532096 i_blocks 45063 00000020:00200000:2.0:1789760776.705522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760776.705541:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760776.705582:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22532096, count: 4096 00000080:00200000:2.0:1789760776.705592:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22532096, bytes: 4096 00000080:00200000:2.0:1789760776.705608:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760776.705740:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760776.705742:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22532096, 22536192) 0 0 00000080:00200000:2.0:1789760776.705747:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760776.705764:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5501, 5501] 00020000:00200000:2.0:1789760776.705774:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22532096 chunk: [22532096, 22536192) 22536192, 4096 00020000:00200000:2.0:1789760776.705778:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760776.705801:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5501, 5501] 00000008:00010000:2.0:1789760776.705820:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760776.705825:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760776.705836:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22532096, 22536192) append 0 append lockpos 0 00000080:00200000:2.0:1789760776.705851:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22532096=0x157d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760776.705924:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760776.705927:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22536192 00000020:00010000:2.0:1789760776.705935:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760776.705939:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.705941:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760776.705945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760776.705948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760776.705957:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760776.705964:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760776.705996:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760776.706112:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 22532096<>22536192 00000100:00100000:2.0:1789760776.706184:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760776.706209:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507339392:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760776.738353:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:2.0:1789760776.738372:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760776.738428:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760776.738444:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:2.0:1789760776.738475:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507339392:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760776.740551:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507339520:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760776.784631:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5502, 5502] 00000080:00200000:1.0:1789760776.784659:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.784665:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.784682:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22540288 i_blocks 45063 00000020:00200000:1.0:1789760776.784685:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.784695:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.784738:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22540288, count: 4096 00000080:00200000:1.0:1789760776.784745:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22540288, bytes: 4096 00000080:00200000:1.0:1789760776.784756:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.784766:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.784769:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22540288, 22544384) 0 0 00000080:00200000:1.0:1789760776.784773:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.784790:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5503, 5503] 00020000:00200000:1.0:1789760776.784798:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22540288 chunk: [22540288, 22544384) 22544384, 4096 00020000:00200000:1.0:1789760776.784803:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.784821:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5503, 5503] 00000008:00010000:1.0:1789760776.784841:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.784847:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.784858:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22540288, 22544384) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.784871:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22540288=0x157f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.784931:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.784934:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22544384 00000020:00010000:1.0:1789760776.784942:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.784945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.784948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.784951:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.784954:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.784962:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.784969:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.784987:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760776.785088:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 22540288<>22544384 00000080:00200000:1.0:1789760776.844421:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5503, 5503] 00000080:00200000:1.0:1789760776.844440:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.844447:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.844463:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22544384 i_blocks 45063 00000020:00200000:1.0:1789760776.844467:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.844477:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.844529:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22544384, count: 4096 00000080:00200000:1.0:1789760776.844539:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22544384, bytes: 4096 00000080:00200000:1.0:1789760776.844549:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.844559:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.844561:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22544384, 22548480) 0 0 00000080:00200000:1.0:1789760776.844567:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.844586:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5504, 5504] 00020000:00200000:1.0:1789760776.844595:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22544384 chunk: [22544384, 22548480) 22548480, 4096 00020000:00200000:1.0:1789760776.844599:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.844619:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5504, 5504] 00000008:00010000:1.0:1789760776.844638:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.844645:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.844658:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22544384, 22548480) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.844672:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22544384=0x1580000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.844735:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.844738:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22548480 00000020:00010000:1.0:1789760776.844747:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.844751:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.844754:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.844758:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.844761:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.844769:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.844776:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.897979:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5504, 5504] 00000080:00200000:1.0:1789760776.897996:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.898003:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.898019:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22548480 i_blocks 45063 00000020:00200000:1.0:1789760776.898024:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.898033:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.898071:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22548480, count: 4096 00000080:00200000:1.0:1789760776.898079:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22548480, bytes: 4096 00000080:00200000:1.0:1789760776.898090:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.898099:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.898101:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22548480, 22552576) 0 0 00000080:00200000:1.0:1789760776.898106:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.898125:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5505, 5505] 00020000:00200000:1.0:1789760776.898133:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22548480 chunk: [22548480, 22552576) 22552576, 4096 00020000:00200000:1.0:1789760776.898138:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.898159:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5505, 5505] 00000008:00010000:1.0:1789760776.898178:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.898185:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.898198:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22548480, 22552576) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.898212:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22548480=0x1581000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.898268:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.898272:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22552576 00000020:00010000:1.0:1789760776.898280:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.898284:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.898287:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.898290:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.898303:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.898312:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.898319:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.898336:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760776.898486:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 22548480<>22552576 00000080:00200000:1.0:1789760776.959352:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5505, 5505] 00000080:00200000:1.0:1789760776.959371:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760776.959533:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760776.959550:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22552576 i_blocks 45063 00000020:00200000:1.0:1789760776.959555:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760776.959566:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760776.959595:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22552576, count: 4096 00000080:00200000:1.0:1789760776.959602:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22552576, bytes: 4096 00000080:00200000:1.0:1789760776.959614:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760776.959622:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760776.959624:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22552576, 22556672) 0 0 00000080:00200000:1.0:1789760776.959628:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760776.959643:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5506, 5506] 00020000:00200000:1.0:1789760776.959651:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22552576 chunk: [22552576, 22556672) 22556672, 4096 00020000:00200000:1.0:1789760776.959654:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760776.959671:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5506, 5506] 00000008:00010000:1.0:1789760776.959685:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760776.959690:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760776.959700:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22552576, 22556672) append 0 append lockpos 0 00000080:00200000:1.0:1789760776.959712:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22552576=0x1582000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760776.959918:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760776.959922:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22556672 00000020:00010000:1.0:1789760776.959929:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760776.959932:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.959934:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760776.959937:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760776.959940:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760776.959947:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760776.959953:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760776.959975:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760776.960754:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 22552576<>22556672 00000080:00200000:2.0:1789760777.006130:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5506, 5506] 00000080:00200000:2.0:1789760777.006143:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.006148:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.006160:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22556672 i_blocks 45063 00000020:00200000:2.0:1789760777.006163:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.006236:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.006261:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22556672, count: 4096 00000080:00200000:2.0:1789760777.006267:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22556672, bytes: 4096 00000080:00200000:2.0:1789760777.006275:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.006283:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.006285:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22556672, 22560768) 0 0 00000080:00200000:2.0:1789760777.006289:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.006301:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5507, 5507] 00020000:00200000:2.0:1789760777.006308:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22556672 chunk: [22556672, 22560768) 22560768, 4096 00020000:00200000:2.0:1789760777.006311:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.006328:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5507, 5507] 00000008:00010000:2.0:1789760777.006342:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.006347:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.006355:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22556672, 22560768) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.006368:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22556672=0x1583000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.006555:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.006558:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22560768 00000020:00010000:2.0:1789760777.006566:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.006569:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.006572:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.006576:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.006578:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.006584:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.006591:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.006606:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760777.006688:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 22556672<>22560768 00000100:00100000:2.0:1789760777.046843:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972827 gen 3 00000008:00010000:2.0:1789760777.046879:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:2.0:1789760777.046886:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760777.046894:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760777.046920:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:2.0:1789760777.046943:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507340160:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760777.047587:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507340288:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760777.089036:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5508, 5508] 00000080:00200000:1.0:1789760777.089054:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760777.089060:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760777.089073:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22564864 i_blocks 45063 00000020:00200000:1.0:1789760777.089078:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760777.089089:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760777.089115:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22564864, count: 4096 00000080:00200000:1.0:1789760777.089121:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22564864, bytes: 4096 00000080:00200000:1.0:1789760777.089132:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760777.089139:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760777.089141:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22564864, 22568960) 0 0 00000080:00200000:1.0:1789760777.089144:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760777.089157:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5509, 5509] 00020000:00200000:1.0:1789760777.089164:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22564864 chunk: [22564864, 22568960) 22568960, 4096 00020000:00200000:1.0:1789760777.089167:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760777.089183:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5509, 5509] 00000008:00010000:1.0:1789760777.089197:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760777.089202:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760777.089210:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22564864, 22568960) append 0 append lockpos 0 00000080:00200000:1.0:1789760777.089220:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22564864=0x1585000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760777.089278:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760777.089281:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22568960 00000020:00010000:1.0:1789760777.089289:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760777.089292:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.089295:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.089298:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760777.089300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760777.089307:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760777.089313:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760777.089330:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760777.089765:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 22564864<>22568960 00000100:00100000:1.0:1789760777.089817:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760777.089835:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507340416:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760777.136285:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5509, 5509] 00000080:00200000:2.0:1789760777.136297:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.136303:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.136315:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22568960 i_blocks 45063 00000020:00200000:2.0:1789760777.136318:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.136326:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.136350:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22568960, count: 4096 00000080:00200000:2.0:1789760777.136355:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22568960, bytes: 4096 00000080:00200000:2.0:1789760777.136362:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.136369:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.136552:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22568960, 22573056) 0 0 00000080:00200000:2.0:1789760777.136557:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.136569:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5510, 5510] 00020000:00200000:2.0:1789760777.136575:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22568960 chunk: [22568960, 22573056) 22573056, 4096 00020000:00200000:2.0:1789760777.136579:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.136593:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5510, 5510] 00000008:00010000:2.0:1789760777.136606:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.136610:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.136618:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22568960, 22573056) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.136628:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22568960=0x1586000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.136665:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.136668:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22573056 00000020:00010000:2.0:1789760777.136674:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.136677:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.136680:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.136683:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.136686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.136691:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.136696:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.136710:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760777.136791:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 22568960<>22573056 00000080:00200000:2.0:1789760777.173525:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5510, 5510] 00000080:00200000:2.0:1789760777.173537:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.173542:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.173554:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22573056 i_blocks 45063 00000020:00200000:2.0:1789760777.173557:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.173564:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.173590:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22573056, count: 4096 00000080:00200000:2.0:1789760777.173596:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22573056, bytes: 4096 00000080:00200000:2.0:1789760777.173603:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.173610:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.173612:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22573056, 22577152) 0 0 00000080:00200000:2.0:1789760777.173617:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.173629:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5511, 5511] 00020000:00200000:2.0:1789760777.173636:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22573056 chunk: [22573056, 22577152) 22577152, 4096 00020000:00200000:2.0:1789760777.173640:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.173654:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5511, 5511] 00000008:00010000:2.0:1789760777.173667:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.173671:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.173680:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22573056, 22577152) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.173691:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22573056=0x1587000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.173747:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.173750:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22577152 00000020:00010000:2.0:1789760777.173756:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.173759:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.173762:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.173765:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.173768:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.173774:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.173779:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.173792:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760777.173872:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 22573056<>22577152 00000100:00100000:2.0:1789760777.236796:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972831 gen 3 00000008:00010000:2.0:1789760777.236854:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac300 00000020:00200000:2.0:1789760777.236861:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760777.236869:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760777.237006:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac300 00000100:00100000:2.0:1789760777.237029:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507340672:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760777.303928:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5512, 5512] 00000080:00200000:2.0:1789760777.303941:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.303950:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.303962:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22581248 i_blocks 45063 00000020:00200000:2.0:1789760777.303965:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.303973:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.303999:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22581248, count: 4096 00000080:00200000:2.0:1789760777.304004:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22581248, bytes: 4096 00000080:00200000:2.0:1789760777.304010:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.304016:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.304017:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22581248, 22585344) 0 0 00000080:00200000:2.0:1789760777.304020:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.304031:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5513, 5513] 00020000:00200000:2.0:1789760777.304036:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22581248 chunk: [22581248, 22585344) 22585344, 4096 00020000:00200000:2.0:1789760777.304039:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.304051:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5513, 5513] 00000008:00010000:2.0:1789760777.304061:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.304065:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.304073:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22581248, 22585344) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.304166:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22581248=0x1589000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.304216:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.304218:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22585344 00000020:00010000:2.0:1789760777.304225:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.304334:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.304338:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.304341:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9128 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.304343:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.304351:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.304357:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.304373:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760777.304688:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 22581248<>22585344 00000100:00100000:1.0:1789760777.348079:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507341056:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760777.374605:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507341184:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760777.412702:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972835 gen 3 00000008:00010000:2.0:1789760777.412749:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c780 00000020:00200000:2.0:1789760777.412823:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760777.412832:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760777.412890:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c780 00000100:00100000:2.0:1789760777.412922:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507341184:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760777.413589:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507341312:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760777.458747:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972836 gen 3 00000008:00010000:2.0:1789760777.458840:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5cd80 00000020:00200000:2.0:1789760777.458847:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760777.458856:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760777.458935:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5cd80 00000100:00100000:2.0:1789760777.458959:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507341312:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760777.459768:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507341440:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760777.498846:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5517, 5517] 00000080:00200000:2.0:1789760777.498859:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.498863:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.498874:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22601728 i_blocks 45063 00000020:00200000:2.0:1789760777.498877:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.498885:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.498909:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22601728, count: 4096 00000080:00200000:2.0:1789760777.498914:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22601728, bytes: 4096 00000080:00200000:2.0:1789760777.498921:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.498927:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.498930:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22601728, 22605824) 0 0 00000080:00200000:2.0:1789760777.498934:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.498945:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5518, 5518] 00020000:00200000:2.0:1789760777.498951:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22601728 chunk: [22601728, 22605824) 22605824, 4096 00020000:00200000:2.0:1789760777.498955:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.498967:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5518, 5518] 00000008:00010000:2.0:1789760777.498979:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.498983:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.498991:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22601728, 22605824) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.499000:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22601728=0x158e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.499038:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.499040:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22605824 00000020:00010000:2.0:1789760777.499046:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.499049:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.499052:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.499055:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.499058:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.499063:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.499068:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.499079:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760777.499144:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 22601728<>22605824 00000080:00200000:2.0:1789760777.549434:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5518, 5518] 00000080:00200000:2.0:1789760777.549447:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.549452:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.549463:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22605824 i_blocks 45063 00000020:00200000:2.0:1789760777.549466:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.549474:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.549499:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22605824, count: 4096 00000080:00200000:2.0:1789760777.549505:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22605824, bytes: 4096 00000080:00200000:2.0:1789760777.549512:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.549519:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.549520:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22605824, 22609920) 0 0 00000080:00200000:2.0:1789760777.549524:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.549535:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5519, 5519] 00020000:00200000:2.0:1789760777.549542:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22605824 chunk: [22605824, 22609920) 22609920, 4096 00020000:00200000:2.0:1789760777.549546:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.549559:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5519, 5519] 00000008:00010000:2.0:1789760777.549572:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.549577:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.549586:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22605824, 22609920) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.549596:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22605824=0x158f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.549637:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.549639:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22609920 00000020:00010000:2.0:1789760777.549646:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.549649:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.549653:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.549656:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.549658:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.549664:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.549668:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.549682:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760777.549774:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 22605824<>22609920 00000080:00200000:2.0:1789760777.611670:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5519, 5519] 00000080:00200000:2.0:1789760777.611683:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.611689:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.611700:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22609920 i_blocks 45063 00000020:00200000:2.0:1789760777.611703:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.611712:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.611745:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22609920, count: 4096 00000080:00200000:2.0:1789760777.611758:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22609920, bytes: 4096 00000080:00200000:2.0:1789760777.611766:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.611774:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.611776:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22609920, 22614016) 0 0 00000080:00200000:2.0:1789760777.611780:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.611792:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5520, 5520] 00020000:00200000:2.0:1789760777.611799:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22609920 chunk: [22609920, 22614016) 22614016, 4096 00020000:00200000:2.0:1789760777.611802:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.611818:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5520, 5520] 00000008:00010000:2.0:1789760777.611831:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.611835:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.611844:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22609920, 22614016) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.611854:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22609920=0x1590000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.612015:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.612019:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22614016 00000020:00010000:2.0:1789760777.612026:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.612030:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.612033:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.612036:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.612038:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.612044:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.612050:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.612063:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760777.612141:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 22609920<>22614016 00000080:00200000:2.0:1789760777.684950:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5520, 5520] 00000080:00200000:2.0:1789760777.684972:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760777.684978:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760777.684992:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22614016 i_blocks 45063 00000020:00200000:2.0:1789760777.684996:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760777.685008:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760777.685034:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22614016, count: 4096 00000080:00200000:2.0:1789760777.685041:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22614016, bytes: 4096 00000080:00200000:2.0:1789760777.685052:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760777.685060:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760777.685062:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22614016, 22618112) 0 0 00000080:00200000:2.0:1789760777.685065:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760777.685076:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5521, 5521] 00020000:00200000:2.0:1789760777.685083:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22614016 chunk: [22614016, 22618112) 22618112, 4096 00020000:00200000:2.0:1789760777.685087:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760777.685103:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5521, 5521] 00000008:00010000:2.0:1789760777.685117:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760777.685123:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760777.685132:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22614016, 22618112) append 0 append lockpos 0 00000080:00200000:2.0:1789760777.685143:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22614016=0x1591000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760777.685200:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760777.685203:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22618112 00000020:00010000:2.0:1789760777.685210:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760777.685213:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.690473:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760777.690479:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760777.690482:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760777.690496:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760777.690507:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760777.690548:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760777.690637:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 22614016<>22618112 00000100:00100000:2.0:1789760777.690680:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760777.690698:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507341952:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760777.746362:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972841 gen 3 00000008:00010000:2.0:1789760777.746534:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac900 00000020:00200000:2.0:1789760777.746541:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760777.746549:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760777.746635:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac900 00000100:00100000:2.0:1789760777.746659:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507341952:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760777.747743:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507342080:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760777.862169:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5523, 5523] 00000080:00200000:1.0:1789760777.862188:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760777.862194:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760777.862210:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22626304 i_blocks 45063 00000020:00200000:1.0:1789760777.862214:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760777.862225:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760777.862263:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22626304, count: 4096 00000080:00200000:1.0:1789760777.862271:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22626304, bytes: 4096 00000080:00200000:1.0:1789760777.862282:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760777.862291:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760777.862294:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22626304, 22630400) 0 0 00000080:00200000:1.0:1789760777.862299:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760777.862316:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5524, 5524] 00020000:00200000:1.0:1789760777.862325:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22626304 chunk: [22626304, 22630400) 22630400, 4096 00020000:00200000:1.0:1789760777.862330:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760777.862351:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5524, 5524] 00000008:00010000:1.0:1789760777.862369:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760777.862374:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760777.862484:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22626304, 22630400) append 0 append lockpos 0 00000080:00200000:1.0:1789760777.862501:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22626304=0x1594000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760777.862567:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760777.862570:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22630400 00000020:00010000:1.0:1789760777.862580:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760777.862584:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.862587:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.862591:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760777.862594:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760777.862604:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760777.862611:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760777.862810:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760777.862930:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 22626304<>22630400 00000080:00200000:1.0:1789760777.909995:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5524, 5524] 00000080:00200000:1.0:1789760777.910010:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760777.910016:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760777.910029:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22630400 i_blocks 45063 00000020:00200000:1.0:1789760777.910032:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760777.910042:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760777.910070:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22630400, count: 4096 00000080:00200000:1.0:1789760777.910076:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22630400, bytes: 4096 00000080:00200000:1.0:1789760777.910086:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760777.910093:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760777.910095:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22630400, 22634496) 0 0 00000080:00200000:1.0:1789760777.910099:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760777.910243:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5525, 5525] 00020000:00200000:1.0:1789760777.910251:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22630400 chunk: [22630400, 22634496) 22634496, 4096 00020000:00200000:1.0:1789760777.910254:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760777.910333:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5525, 5525] 00000008:00010000:1.0:1789760777.910350:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760777.910355:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760777.910365:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22630400, 22634496) append 0 append lockpos 0 00000080:00200000:1.0:1789760777.910407:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22630400=0x1595000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760777.910452:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760777.910455:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22634496 00000020:00010000:1.0:1789760777.910462:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760777.910466:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.910468:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.910472:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760777.910474:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760777.910480:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760777.910486:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760777.910501:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760777.910583:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 22630400<>22634496 00000080:00200000:1.0:1789760777.956236:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5525, 5525] 00000080:00200000:1.0:1789760777.956249:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760777.956254:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760777.956267:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22634496 i_blocks 45063 00000020:00200000:1.0:1789760777.956270:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760777.956278:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760777.956304:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22634496, count: 4096 00000080:00200000:1.0:1789760777.956310:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22634496, bytes: 4096 00000080:00200000:1.0:1789760777.956320:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760777.956327:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760777.956330:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22634496, 22638592) 0 0 00000080:00200000:1.0:1789760777.956333:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760777.956346:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5526, 5526] 00020000:00200000:1.0:1789760777.956352:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22634496 chunk: [22634496, 22638592) 22638592, 4096 00020000:00200000:1.0:1789760777.956356:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760777.956370:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5526, 5526] 00000008:00010000:1.0:1789760777.956403:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760777.956409:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760777.956417:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22634496, 22638592) append 0 append lockpos 0 00000080:00200000:1.0:1789760777.956428:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22634496=0x1596000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760777.956471:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760777.956473:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22638592 00000020:00010000:1.0:1789760777.956481:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760777.956485:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.956488:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760777.956491:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760777.956494:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760777.956583:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760777.956589:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760777.956603:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760777.956678:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 22634496<>22638592 00000100:00100000:1.0:1789760777.988149:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2417:1876699507342720:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760777.989710:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507342848:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760777.989738:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507343104:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760777.989758:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507342976:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760777.989769:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507343232:192.168.203.141@tcp:400:kworker.0 00000008:00010000:1.0:1789760778.006456:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760778.006473:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.006484:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.006509:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760778.006543:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507342592:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760778.007891:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507343360:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760778.062856:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972847 gen 3 00000008:00010000:2.0:1789760778.062941:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c300 00000020:00200000:2.0:1789760778.062950:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760778.062961:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760778.063035:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c300 00000100:00100000:2.0:1789760778.063066:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507343360:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.066269:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507343488:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.116920:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972848 gen 3 00000008:00010000:1.0:1789760778.116972:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760778.116980:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.116993:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.117075:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760778.117108:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507343488:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.119911:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507343616:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760778.163707:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760778.163727:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.163737:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.163792:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760778.163816:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507343616:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.195172:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972850 gen 3 00000008:00010000:1.0:1789760778.195221:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760778.195228:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.195238:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.195256:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760778.195280:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507343744:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.246508:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972851 gen 3 00000008:00010000:1.0:1789760778.246571:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760778.246580:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.246594:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.246620:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760778.246651:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507343872:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.248055:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507344000:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.283701:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972852 gen 3 00000008:00010000:1.0:1789760778.283771:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760778.283779:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.283792:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.283867:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760778.283911:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507344000:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.285303:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507344128:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.313047:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972853 gen 3 00000008:00010000:1.0:1789760778.313103:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760778.313112:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.313126:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.313148:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760778.313198:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507344128:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760778.313524:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5533, 5533] 00000080:00200000:2.0:1789760778.313541:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760778.313548:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760778.313565:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22667264 i_blocks 45063 00000020:00200000:2.0:1789760778.313570:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760778.313581:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760778.313627:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22667264, count: 4096 00000080:00200000:2.0:1789760778.313634:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22667264, bytes: 4096 00000080:00200000:2.0:1789760778.313646:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760778.313655:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760778.313658:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22667264, 22671360) 0 0 00000080:00200000:2.0:1789760778.313663:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760778.313678:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5534, 5534] 00020000:00200000:2.0:1789760778.313687:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22667264 chunk: [22667264, 22671360) 22671360, 4096 00020000:00200000:2.0:1789760778.313691:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760778.313721:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5534, 5534] 00000008:00010000:2.0:1789760778.313740:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760778.313746:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760778.313759:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22667264, 22671360) append 0 append lockpos 0 00000080:00200000:2.0:1789760778.313773:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22667264=0x159e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760778.313830:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760778.313833:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22671360 00000020:00010000:2.0:1789760778.313840:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760778.313844:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760778.313847:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760778.313850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760778.313853:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760778.313860:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760778.313867:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760778.313882:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760778.313985:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 22667264<>22671360 00000080:00200000:2.0:1789760778.344088:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5534, 5534] 00000080:00200000:2.0:1789760778.344109:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760778.344116:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760778.344132:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22671360 i_blocks 45063 00000020:00200000:2.0:1789760778.344137:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760778.344148:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760778.344191:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22671360, count: 4096 00000080:00200000:2.0:1789760778.344199:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22671360, bytes: 4096 00000080:00200000:2.0:1789760778.344209:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760778.344218:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760778.344220:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22671360, 22675456) 0 0 00000080:00200000:2.0:1789760778.344225:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760778.344243:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5535, 5535] 00020000:00200000:2.0:1789760778.344252:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22671360 chunk: [22671360, 22675456) 22675456, 4096 00020000:00200000:2.0:1789760778.344257:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760778.344278:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5535, 5535] 00000008:00010000:2.0:1789760778.344295:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760778.344301:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760778.344313:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22671360, 22675456) append 0 append lockpos 0 00000080:00200000:2.0:1789760778.344327:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22671360=0x159f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760778.344441:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760778.344444:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22675456 00000020:00010000:2.0:1789760778.344453:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760778.344456:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760778.344460:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760778.344464:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760778.344467:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760778.344475:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760778.344482:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760778.344501:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760778.344602:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 22671360<>22675456 00000080:00200000:2.0:1789760778.385853:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5535, 5535] 00000080:00200000:2.0:1789760778.385871:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760778.385878:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760778.385895:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22675456 i_blocks 45063 00000020:00200000:2.0:1789760778.385899:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760778.385909:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760778.386103:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22675456, count: 4096 00000080:00200000:2.0:1789760778.386113:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22675456, bytes: 4096 00000080:00200000:2.0:1789760778.386127:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760778.386136:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760778.386138:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22675456, 22679552) 0 0 00000080:00200000:2.0:1789760778.386142:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760778.386159:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5536, 5536] 00020000:00200000:2.0:1789760778.386168:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22675456 chunk: [22675456, 22679552) 22679552, 4096 00020000:00200000:2.0:1789760778.386172:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760778.386194:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5536, 5536] 00000008:00010000:2.0:1789760778.386212:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760778.386218:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760778.386230:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22675456, 22679552) append 0 append lockpos 0 00000080:00200000:2.0:1789760778.386244:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22675456=0x15a0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760778.386477:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760778.386480:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22679552 00000020:00010000:2.0:1789760778.386489:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760778.386493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760778.386496:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760778.386499:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760778.386503:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760778.386511:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760778.386518:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760778.386539:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760778.388779:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 22675456<>22679552 00000100:00100000:2.0:1789760778.424228:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972856 gen 3 00000008:00010000:2.0:1789760778.424279:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760778.424287:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760778.424299:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760778.424327:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760778.424361:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507344512:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760778.424532:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5536, 5536] 00000080:00200000:1.0:1789760778.424550:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760778.424556:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760778.424572:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22679552 i_blocks 45063 00000020:00200000:1.0:1789760778.424575:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760778.424585:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760778.424618:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22679552, count: 4096 00000080:00200000:1.0:1789760778.424625:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22679552, bytes: 4096 00000080:00200000:1.0:1789760778.424640:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760778.424660:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760778.424663:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22679552, 22683648) 0 0 00000080:00200000:1.0:1789760778.424667:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760778.424683:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5537, 5537] 00020000:00200000:1.0:1789760778.424691:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22679552 chunk: [22679552, 22683648) 22683648, 4096 00020000:00200000:1.0:1789760778.424695:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760778.424734:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5537, 5537] 00000008:00010000:1.0:1789760778.424753:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760778.424759:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760778.424769:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22679552, 22683648) append 0 append lockpos 0 00000080:00200000:1.0:1789760778.424782:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22679552=0x15a1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760778.424838:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760778.424841:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22683648 00000020:00010000:1.0:1789760778.424848:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760778.424851:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760778.424854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760778.424858:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760778.424861:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760778.424868:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760778.424875:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760778.424891:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760778.424995:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22679552<>22683648 00000100:00100000:1.0:1789760778.425030:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760778.425052:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507344640:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.454348:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972857 gen 3 00000008:00010000:1.0:1789760778.454461:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760778.454469:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.454481:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.454500:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760778.454527:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507344640:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.489519:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972858 gen 3 00000008:00010000:1.0:1789760778.489574:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760778.489583:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.489594:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.489613:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760778.489645:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507344768:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.490373:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760778.490476:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507344896:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.525575:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972859 gen 3 00000008:00010000:1.0:1789760778.525633:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760778.525641:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.525656:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.525675:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760778.525719:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507344896:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.526977:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507345024:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760778.555467:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760778.555483:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.555495:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.555507:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760778.555537:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507345024:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.556347:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507345152:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.582433:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972861 gen 3 00000008:00010000:1.0:1789760778.582488:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760778.582495:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.582505:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.582571:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760778.582595:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507345152:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.583354:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507345280:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.616974:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972862 gen 3 00000008:00010000:1.0:1789760778.617027:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760778.617036:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.617048:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.617070:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760778.617103:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507345280:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760778.646183:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760778.646201:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.646216:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.646235:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760778.646267:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507345408:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.683559:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972864 gen 3 00000008:00010000:1.0:1789760778.683624:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760778.683634:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.683645:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.683666:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760778.683694:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507345536:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.684557:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507345664:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.715440:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972865 gen 3 00000008:00010000:1.0:1789760778.715478:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760778.715484:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.715494:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.715602:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760778.715625:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507345664:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.716044:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760778.716058:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507345792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.740954:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972866 gen 3 00000008:00010000:1.0:1789760778.740991:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760778.740998:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.741007:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.741026:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760778.741049:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507345792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.742238:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760778.742255:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507345920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.775624:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972867 gen 3 00000008:00010000:1.0:1789760778.775659:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760778.775666:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.775675:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.775744:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760778.775769:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507345920:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.828057:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972868 gen 3 00000080:00200000:1.0:1789760778.828618:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5548, 5548] 00000080:00200000:1.0:1789760778.828630:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760778.828635:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000100:00100000:1.0:1789760778.829560:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760778.829576:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346176:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.871065:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972869 gen 3 00000008:00010000:1.0:1789760778.871102:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760778.871108:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.871118:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.871282:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760778.871307:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346176:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.872164:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346304:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.914283:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972870 gen 3 00000008:00010000:1.0:1789760778.914319:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760778.914326:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.914343:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.914361:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760778.914510:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346304:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.915537:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507346432:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.955059:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972871 gen 3 00000008:00010000:1.0:1789760778.955095:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760778.955102:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760778.955113:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760778.955234:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760778.955273:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507346432:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760778.956147:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346560:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.008484:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972872 gen 3 00000008:00010000:1.0:1789760779.008532:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760779.008540:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.008552:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.008612:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760779.008650:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346560:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.012316:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507346688:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760779.074669:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760779.074685:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.074699:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.074765:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760779.074797:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507346688:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.075407:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.114510:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972873 gen 3 00000100:00100000:1.0:1789760779.114560:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b8e00 x1876699507346816/t4294972874(4294972874) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760795 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760779.114623:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760779.114631:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.114643:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.114661:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760779.114687:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507346816:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.115807:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507346944:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.187477:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972875 gen 3 00000008:00010000:1.0:1789760779.187524:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760779.187530:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.187540:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.187560:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760779.187584:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507346944:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.188649:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760779.188665:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347072:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.241469:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972876 gen 3 00000008:00010000:1.0:1789760779.241503:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760779.241509:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.241518:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.241536:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760779.241557:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347072:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.242977:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347200:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.279990:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972877 gen 3 00000008:00010000:1.0:1789760779.280026:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760779.280033:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.280042:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.280058:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760779.280080:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347200:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.281792:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760779.281827:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347328:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760779.315574:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760779.315590:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.315603:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.315624:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760779.315655:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347328:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.316483:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.383897:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972879 gen 3 00000008:00010000:1.0:1789760779.383966:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760779.383975:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.383989:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.384060:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760779.384094:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347456:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.389323:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507347584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.443005:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972880 gen 3 00000008:00010000:1.0:1789760779.443059:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760779.443068:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.443084:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.443109:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760779.443140:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507347584:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.445024:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347712:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.486553:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972881 gen 3 00000008:00010000:1.0:1789760779.486603:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760779.486622:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.486636:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.486658:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760779.486689:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347712:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.490331:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507347840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.528177:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972882 gen 3 00000008:00010000:1.0:1789760779.528227:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760779.528236:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.528247:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.528431:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760779.528463:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507347840:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.529286:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347968:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760779.584606:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760779.584619:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.584631:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.584651:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760779.584683:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507347968:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.586651:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348096:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.616842:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972883 gen 3 00000100:00100000:1.0:1789760779.616886:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b9180 x1876699507348096/t4294972884(4294972884) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760795 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760779.617205:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760779.617213:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.617226:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.617251:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760779.617283:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348096:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.619664:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760779.619692:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348224:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.654718:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972885 gen 3 00000008:00010000:1.0:1789760779.654795:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760779.654804:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.654815:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.654836:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760779.654867:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348224:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760779.655428:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7480 00000008:00100000:1.0:1789760779.655535:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22798336<>22802432 00000100:00100000:1.0:1789760779.655593:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348352:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760779.684000:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760779.684016:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.684030:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.684056:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:1.0:1789760779.684091:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348352:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.685042:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507348480:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.729075:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972887 gen 3 00000008:00010000:1.0:1789760779.729142:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760779.729151:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.729162:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.729230:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760779.729260:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507348480:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.770975:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972888 gen 3 00000008:00010000:1.0:1789760779.771031:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760779.771040:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.771054:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.771077:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760779.771111:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507348608:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.771818:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507348736:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.804457:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972889 gen 3 00000008:00010000:1.0:1789760779.804490:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760779.804497:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.804507:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.804717:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760779.804742:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507348736:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.807839:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348864:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760779.842017:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972890 gen 3 00000008:00010000:1.0:1789760779.842062:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760779.842069:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760779.842082:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760779.842112:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760779.842146:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507348864:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760779.842302:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5570, 5570] 00000080:00200000:2.0:1789760779.842329:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760779.842337:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760779.842353:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22818816 i_blocks 45063 00000020:00200000:2.0:1789760779.842358:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760779.842372:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760779.842450:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22818816, count: 4096 00000080:00200000:2.0:1789760779.842459:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22818816, bytes: 4096 00000080:00200000:2.0:1789760779.842475:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760779.842485:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760779.842487:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22818816, 22822912) 0 0 00000080:00200000:2.0:1789760779.842492:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760779.842509:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5571, 5571] 00020000:00200000:2.0:1789760779.842519:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22818816 chunk: [22818816, 22822912) 22822912, 4096 00020000:00200000:2.0:1789760779.842524:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760779.842549:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5571, 5571] 00000008:00010000:2.0:1789760779.842570:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760779.842577:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760779.842590:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22818816, 22822912) append 0 append lockpos 0 00000080:00200000:2.0:1789760779.842607:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22818816=0x15c3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760779.842674:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760779.842677:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22822912 00000020:00010000:2.0:1789760779.842686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760779.842690:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760779.842693:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760779.842775:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760779.842778:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760779.842789:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760779.842796:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760779.842815:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760779.842930:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 22818816<>22822912 00000080:00200000:2.0:1789760779.910654:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5571, 5571] 00000080:00200000:2.0:1789760779.910684:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760779.910692:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760779.910720:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22822912 i_blocks 45063 00000020:00200000:2.0:1789760779.910726:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760779.910743:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760779.910779:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22822912, count: 4096 00000080:00200000:2.0:1789760779.910787:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22822912, bytes: 4096 00000080:00200000:2.0:1789760779.910801:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760779.910811:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760779.910813:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22822912, 22827008) 0 0 00000080:00200000:2.0:1789760779.910818:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760779.910835:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5572, 5572] 00020000:00200000:2.0:1789760779.910844:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22822912 chunk: [22822912, 22827008) 22827008, 4096 00020000:00200000:2.0:1789760779.910849:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760779.910873:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5572, 5572] 00000008:00010000:2.0:1789760779.910894:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760779.910900:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760779.910913:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22822912, 22827008) append 0 append lockpos 0 00000080:00200000:2.0:1789760779.910928:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22822912=0x15c4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760779.911001:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760779.911003:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22827008 00000020:00010000:2.0:1789760779.911012:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760779.911016:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760779.911019:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760779.911023:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760779.911026:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760779.911036:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760779.911043:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760779.911067:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760779.911185:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 22822912<>22827008 00000080:00200000:2.0:1789760779.957287:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5572, 5572] 00000080:00200000:2.0:1789760779.957300:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760779.957305:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760779.957317:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22827008 i_blocks 45063 00000020:00200000:2.0:1789760779.957320:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760779.957328:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760779.957354:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22827008, count: 4096 00000080:00200000:2.0:1789760779.957360:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22827008, bytes: 4096 00000080:00200000:2.0:1789760779.957367:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760779.957374:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760779.957411:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22827008, 22831104) 0 0 00000080:00200000:2.0:1789760779.957415:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760779.957429:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5573, 5573] 00020000:00200000:2.0:1789760779.957436:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22827008 chunk: [22827008, 22831104) 22831104, 4096 00020000:00200000:2.0:1789760779.957440:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760779.957454:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5573, 5573] 00000008:00010000:2.0:1789760779.957467:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760779.957472:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760779.957482:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22827008, 22831104) append 0 append lockpos 0 00000080:00200000:2.0:1789760779.957492:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22827008=0x15c5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760779.957534:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760779.957536:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22831104 00000020:00010000:2.0:1789760779.957543:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760779.957546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760779.957549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760779.957551:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760779.957554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760779.957561:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760779.957566:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760779.957579:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760779.957656:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022d80 - object 0x280000401:3 offset 22827008<>22831104 00000080:00200000:2.0:1789760780.000604:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5573, 5573] 00000080:00200000:2.0:1789760780.000617:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.000622:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.000638:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22831104 i_blocks 45063 00000020:00200000:2.0:1789760780.000642:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.000654:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.000679:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22831104, count: 4096 00000080:00200000:2.0:1789760780.000684:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22831104, bytes: 4096 00000080:00200000:2.0:1789760780.000691:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.000707:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.000709:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22831104, 22835200) 0 0 00000080:00200000:2.0:1789760780.000713:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.000729:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5574, 5574] 00020000:00200000:2.0:1789760780.000735:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22831104 chunk: [22831104, 22835200) 22835200, 4096 00020000:00200000:2.0:1789760780.000740:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.000754:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5574, 5574] 00000008:00010000:2.0:1789760780.000766:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.000770:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.000779:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22831104, 22835200) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.000790:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22831104=0x15c6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.000838:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.000841:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22835200 00000020:00010000:2.0:1789760780.000847:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.000850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.000853:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.000857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.000859:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.000866:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.000876:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.000889:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760780.000967:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 22831104<>22835200 00000080:00200000:2.0:1789760780.040323:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5574, 5574] 00000080:00200000:2.0:1789760780.040343:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.040349:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.040362:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22835200 i_blocks 45063 00000020:00200000:2.0:1789760780.040367:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.040418:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.040446:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22835200, count: 4096 00000080:00200000:2.0:1789760780.040452:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22835200, bytes: 4096 00000080:00200000:2.0:1789760780.040463:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.040472:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.040474:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22835200, 22839296) 0 0 00000080:00200000:2.0:1789760780.040477:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.040489:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5575, 5575] 00020000:00200000:2.0:1789760780.040496:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22835200 chunk: [22835200, 22839296) 22839296, 4096 00020000:00200000:2.0:1789760780.040501:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.040517:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5575, 5575] 00000008:00010000:2.0:1789760780.040532:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.040536:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.040544:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22835200, 22839296) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.040622:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22835200=0x15c7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.040676:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.040679:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22839296 00000020:00010000:2.0:1789760780.040686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.040689:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.040692:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.040695:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.040712:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.040719:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.040725:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.040744:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760780.040825:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 22835200<>22839296 00000100:00100000:1.0:1789760780.067751:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760780.067871:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507349632:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.100205:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972896 gen 3 00000008:00010000:1.0:1789760780.100255:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760780.100264:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.100276:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.100305:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760780.100335:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507349632:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760780.100492:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5576, 5576] 00000080:00200000:2.0:1789760780.100508:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.100514:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.100528:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22843392 i_blocks 45063 00000020:00200000:2.0:1789760780.100532:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.100543:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.100578:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22843392, count: 4096 00000080:00200000:2.0:1789760780.100586:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22843392, bytes: 4096 00000080:00200000:2.0:1789760780.100597:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.100606:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.100609:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22843392, 22847488) 0 0 00000080:00200000:2.0:1789760780.100614:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.100632:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5577, 5577] 00020000:00200000:2.0:1789760780.100640:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22843392 chunk: [22843392, 22847488) 22847488, 4096 00020000:00200000:2.0:1789760780.100645:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.100666:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5577, 5577] 00000008:00010000:2.0:1789760780.100690:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.100704:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.100716:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22843392, 22847488) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.100731:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22843392=0x15c9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.100786:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.100789:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22847488 00000020:00010000:2.0:1789760780.100797:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.100801:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.100804:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.100807:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.100809:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.100817:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.100824:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.100839:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760780.100960:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 22843392<>22847488 00000080:00200000:2.0:1789760780.145272:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5577, 5577] 00000080:00200000:2.0:1789760780.145289:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.145296:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.145313:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22847488 i_blocks 45063 00000020:00200000:2.0:1789760780.145317:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.145328:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.145446:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22847488, count: 4096 00000080:00200000:2.0:1789760780.145454:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22847488, bytes: 4096 00000080:00200000:2.0:1789760780.145466:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.145476:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.145479:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22847488, 22851584) 0 0 00000080:00200000:2.0:1789760780.145484:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.145501:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5578, 5578] 00020000:00200000:2.0:1789760780.145510:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22847488 chunk: [22847488, 22851584) 22851584, 4096 00020000:00200000:2.0:1789760780.145515:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.145535:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5578, 5578] 00000008:00010000:2.0:1789760780.145553:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.145559:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.145571:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22847488, 22851584) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.145586:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22847488=0x15ca000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.145775:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.145778:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22851584 00000020:00010000:2.0:1789760780.145788:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.145791:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.145794:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.145798:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.145801:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.145810:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.145817:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.145836:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760780.145948:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 22847488<>22851584 00000080:00200000:2.0:1789760780.185038:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5578, 5578] 00000080:00200000:2.0:1789760780.185067:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.185075:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.185101:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22851584 i_blocks 45063 00000020:00200000:2.0:1789760780.185108:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.185127:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.185521:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22851584, count: 4096 00000080:00200000:2.0:1789760780.185533:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22851584, bytes: 4096 00000080:00200000:2.0:1789760780.185547:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.185557:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.185559:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22851584, 22855680) 0 0 00000080:00200000:2.0:1789760780.185563:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.185579:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5579, 5579] 00020000:00200000:2.0:1789760780.185588:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22851584 chunk: [22851584, 22855680) 22855680, 4096 00020000:00200000:2.0:1789760780.185592:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.185863:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5579, 5579] 00000008:00010000:2.0:1789760780.185883:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.185889:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.185902:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22851584, 22855680) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.185917:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22851584=0x15cb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.185986:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.185989:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22855680 00000020:00010000:2.0:1789760780.185997:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.186000:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.186009:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.186012:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.186015:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.186025:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.186032:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.186057:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760780.186257:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 22851584<>22855680 00000080:00200000:2.0:1789760780.220972:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5579, 5579] 00000080:00200000:2.0:1789760780.220990:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.220997:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.221013:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22855680 i_blocks 45063 00000020:00200000:2.0:1789760780.221018:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.221028:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.221061:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22855680, count: 4096 00000080:00200000:2.0:1789760780.221069:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22855680, bytes: 4096 00000080:00200000:2.0:1789760780.221081:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.221090:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.221092:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22855680, 22859776) 0 0 00000080:00200000:2.0:1789760780.221097:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.221114:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5580, 5580] 00020000:00200000:2.0:1789760780.221123:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22855680 chunk: [22855680, 22859776) 22859776, 4096 00020000:00200000:2.0:1789760780.221128:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.221148:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5580, 5580] 00000008:00010000:2.0:1789760780.221168:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.221174:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.221199:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22855680, 22859776) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.221214:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22855680=0x15cc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.221275:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.221277:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22859776 00000020:00010000:2.0:1789760780.221286:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.221289:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.221292:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.221296:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.221299:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.221308:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.221314:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.221332:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760780.221561:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 22855680<>22859776 00000080:00200000:2.0:1789760780.256747:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5580, 5580] 00000080:00200000:2.0:1789760780.256764:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.256770:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.256785:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22859776 i_blocks 45063 00000020:00200000:2.0:1789760780.256789:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.256799:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.256832:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22859776, count: 4096 00000080:00200000:2.0:1789760780.256840:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22859776, bytes: 4096 00000080:00200000:2.0:1789760780.256849:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.256858:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.256861:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22859776, 22863872) 0 0 00000080:00200000:2.0:1789760780.256865:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.256882:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5581, 5581] 00020000:00200000:2.0:1789760780.256890:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22859776 chunk: [22859776, 22863872) 22863872, 4096 00020000:00200000:2.0:1789760780.256894:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.256914:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5581, 5581] 00000008:00010000:2.0:1789760780.256932:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.256937:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.256949:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22859776, 22863872) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.256963:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22859776=0x15cd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.257017:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.257020:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22863872 00000020:00010000:2.0:1789760780.257029:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.257032:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.257035:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.257038:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.257041:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.257049:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.257056:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.257072:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760780.257179:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 22859776<>22863872 00000100:00100000:1.0:1789760780.344545:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972902 gen 3 00000008:00010000:1.0:1789760780.344628:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760780.344637:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.344650:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.344672:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760780.344711:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507350400:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760780.344745:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5582, 5582] 00000080:00200000:2.0:1789760780.344763:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.344769:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.344787:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22867968 i_blocks 45063 00000020:00200000:2.0:1789760780.344791:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.344802:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.344839:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22867968, count: 4096 00000080:00200000:2.0:1789760780.344848:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22867968, bytes: 4096 00000080:00200000:2.0:1789760780.344861:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.344872:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.344875:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22867968, 22872064) 0 0 00000080:00200000:2.0:1789760780.344880:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.344899:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5583, 5583] 00020000:00200000:2.0:1789760780.344908:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22867968 chunk: [22867968, 22872064) 22872064, 4096 00020000:00200000:2.0:1789760780.344913:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.344934:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5583, 5583] 00000008:00010000:2.0:1789760780.344954:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.344960:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.344973:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22867968, 22872064) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.344988:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22867968=0x15cf000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.345049:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.345052:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22872064 00000020:00010000:2.0:1789760780.345061:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.345064:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.345067:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.345071:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.345074:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.345083:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.345090:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.345105:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760780.345224:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 22867968<>22872064 00000080:00200000:2.0:1789760780.373020:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5583, 5583] 00000080:00200000:2.0:1789760780.373038:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.373044:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.373060:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22872064 i_blocks 45063 00000020:00200000:2.0:1789760780.373064:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.373076:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.373110:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22872064, count: 4096 00000080:00200000:2.0:1789760780.373117:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22872064, bytes: 4096 00000080:00200000:2.0:1789760780.373131:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.373250:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.373253:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22872064, 22876160) 0 0 00000080:00200000:2.0:1789760780.373257:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.373273:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5584, 5584] 00020000:00200000:2.0:1789760780.373280:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22872064 chunk: [22872064, 22876160) 22876160, 4096 00020000:00200000:2.0:1789760780.373284:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.373305:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5584, 5584] 00000008:00010000:2.0:1789760780.373323:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.373329:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.373340:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22872064, 22876160) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.373353:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22872064=0x15d0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.373435:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.373438:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22876160 00000020:00010000:2.0:1789760780.373446:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.373450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.373453:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80385a8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.373456:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.373459:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.373466:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.373472:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.373493:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760780.373599:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022d80 - object 0x280000401:3 offset 22872064<>22876160 00000008:00010000:2.0:1789760780.417823:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760780.417839:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760780.417851:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760780.417873:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760780.417901:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507350656:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.419329:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507350784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.447533:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972905 gen 3 00000008:00010000:1.0:1789760780.447605:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760780.447614:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.447626:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.447652:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760780.447689:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507350784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.448588:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507350912:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.480557:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972906 gen 3 00000008:00010000:1.0:1789760780.480603:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760780.480611:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.480623:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.480679:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760780.480716:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507350912:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760780.488480:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760780.488492:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760780.488495:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22884352, 22888448) 0 0 00000080:00200000:1.0:1789760780.488502:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760780.488522:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5587, 5587] 00020000:00200000:1.0:1789760780.488531:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22884352 chunk: [22884352, 22888448) 22888448, 4096 00020000:00200000:1.0:1789760780.488537:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760780.488563:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5587, 5587] 00000008:00010000:1.0:1789760780.488584:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760780.488590:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760780.488603:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22884352, 22888448) append 0 append lockpos 0 00000080:00200000:1.0:1789760780.488618:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22884352=0x15d3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760780.488691:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760780.490989:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22888448 00000020:00010000:1.0:1789760780.491010:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760780.491015:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.491021:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.491026:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760780.491029:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760780.491103:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760780.491116:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760780.491217:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760780.491488:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22884352<>22888448 00000100:00100000:1.0:1789760780.491551:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760780.491576:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507351040:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.531476:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972907 gen 3 00000008:00010000:1.0:1789760780.531534:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760780.531542:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.531554:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.532204:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760780.532242:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507351040:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760780.532672:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5587, 5587] 00000080:00200000:2.0:1789760780.532691:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.532706:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.532722:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22888448 i_blocks 45063 00000020:00200000:2.0:1789760780.532726:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.532802:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.532835:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22888448, count: 4096 00000080:00200000:2.0:1789760780.532842:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22888448, bytes: 4096 00000080:00200000:2.0:1789760780.532856:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.532865:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.532867:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22888448, 22892544) 0 0 00000080:00200000:2.0:1789760780.532872:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.532888:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5588, 5588] 00020000:00200000:2.0:1789760780.532954:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22888448 chunk: [22888448, 22892544) 22892544, 4096 00020000:00200000:2.0:1789760780.532959:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.532980:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5588, 5588] 00000008:00010000:2.0:1789760780.532998:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.533004:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.533015:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22888448, 22892544) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.533029:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22888448=0x15d4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.533088:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.533091:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22892544 00000020:00010000:2.0:1789760780.533100:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.533103:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.533106:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.533110:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.533113:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.533120:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.533127:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.533252:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760780.533357:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 22888448<>22892544 00000080:00200000:2.0:1789760780.569484:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5588, 5588] 00000080:00200000:2.0:1789760780.569501:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.569508:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.569524:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22892544 i_blocks 45063 00000020:00200000:2.0:1789760780.569528:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.569539:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.569574:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22892544, count: 4096 00000080:00200000:2.0:1789760780.569582:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22892544, bytes: 4096 00000080:00200000:2.0:1789760780.569592:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.569601:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.569604:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22892544, 22896640) 0 0 00000080:00200000:2.0:1789760780.569608:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.569628:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5589, 5589] 00020000:00200000:2.0:1789760780.569635:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22892544 chunk: [22892544, 22896640) 22896640, 4096 00020000:00200000:2.0:1789760780.569641:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.569662:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5589, 5589] 00000008:00010000:2.0:1789760780.569702:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.569708:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.569720:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22892544, 22896640) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.569734:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22892544=0x15d5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.569794:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.569798:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22896640 00000020:00010000:2.0:1789760780.569806:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.569810:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.569814:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.569817:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.569820:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.569828:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.569837:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.569854:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760780.569973:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 22892544<>22896640 00000080:00200000:2.0:1789760780.628050:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5589, 5589] 00000080:00200000:2.0:1789760780.628067:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.628073:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.628088:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22896640 i_blocks 45063 00000020:00200000:2.0:1789760780.628093:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.628102:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.628139:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22896640, count: 4096 00000080:00200000:2.0:1789760780.628159:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22896640, bytes: 4096 00000080:00200000:2.0:1789760780.628170:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.628180:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.628182:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22896640, 22900736) 0 0 00000080:00200000:2.0:1789760780.628187:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.628206:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5590, 5590] 00020000:00200000:2.0:1789760780.628215:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22896640 chunk: [22896640, 22900736) 22900736, 4096 00020000:00200000:2.0:1789760780.628220:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.628241:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5590, 5590] 00000008:00010000:2.0:1789760780.628262:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.628269:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.628281:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22896640, 22900736) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.628295:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22896640=0x15d6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.628354:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.628357:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22900736 00000020:00010000:2.0:1789760780.628366:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.628369:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.628372:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.628375:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.628776:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.628797:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.628805:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.628826:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760780.628959:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 22896640<>22900736 00000100:00100000:2.0:1789760780.629008:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760780.629028:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507351424:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760780.678099:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972910 gen 3 00000008:00010000:2.0:1789760780.678220:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac900 00000020:00200000:2.0:1789760780.678230:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760780.678243:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760780.678269:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac900 00000100:00100000:2.0:1789760780.678301:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507351424:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760780.678461:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5590, 5590] 00000080:00200000:1.0:1789760780.678478:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760780.678485:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760780.678501:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22900736 i_blocks 45063 00000020:00200000:1.0:1789760780.678505:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760780.678516:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760780.678554:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22900736, count: 4096 00000080:00200000:1.0:1789760780.678563:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22900736, bytes: 4096 00000080:00200000:1.0:1789760780.678575:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760780.678585:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760780.678587:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22900736, 22904832) 0 0 00000080:00200000:1.0:1789760780.678592:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760780.678609:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5591, 5591] 00020000:00200000:1.0:1789760780.678617:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22900736 chunk: [22900736, 22904832) 22904832, 4096 00020000:00200000:1.0:1789760780.678622:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760780.678643:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5591, 5591] 00000008:00010000:1.0:1789760780.678663:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760780.678670:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760780.678682:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22900736, 22904832) append 0 append lockpos 0 00000080:00200000:1.0:1789760780.678707:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22900736=0x15d7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760780.678769:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760780.678772:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22904832 00000020:00010000:1.0:1789760780.678911:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760780.678916:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.678918:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.678922:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760780.678925:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760780.678934:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760780.678941:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760780.678962:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760780.679078:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22900736<>22904832 00000100:00100000:1.0:1789760780.728116:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972911 gen 3 00000008:00010000:1.0:1789760780.728172:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760780.728181:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.728192:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.728220:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760780.728250:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507351552:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760780.728410:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5591, 5591] 00000080:00200000:2.0:1789760780.728426:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.728432:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.728449:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22904832 i_blocks 45063 00000020:00200000:2.0:1789760780.728454:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.728465:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.728500:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22904832, count: 4096 00000080:00200000:2.0:1789760780.728507:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22904832, bytes: 4096 00000080:00200000:2.0:1789760780.728519:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.728528:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.728531:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22904832, 22908928) 0 0 00000080:00200000:2.0:1789760780.728535:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.728554:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5592, 5592] 00020000:00200000:2.0:1789760780.728564:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22904832 chunk: [22904832, 22908928) 22908928, 4096 00020000:00200000:2.0:1789760780.728569:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.728589:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5592, 5592] 00000008:00010000:2.0:1789760780.728608:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.728758:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.728772:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22904832, 22908928) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.728796:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22904832=0x15d8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.728858:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.728861:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22908928 00000020:00010000:2.0:1789760780.728869:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.728873:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.728876:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.728880:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.728883:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.728891:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.728898:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.728917:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760780.729025:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 22904832<>22908928 00000080:00200000:2.0:1789760780.789831:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5592, 5592] 00000080:00200000:2.0:1789760780.789847:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760780.789853:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760780.789868:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22908928 i_blocks 45063 00000020:00200000:2.0:1789760780.789871:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760780.789885:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760780.789918:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22908928, count: 4096 00000080:00200000:2.0:1789760780.789925:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22908928, bytes: 4096 00000080:00200000:2.0:1789760780.789934:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760780.789943:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760780.789946:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22908928, 22913024) 0 0 00000080:00200000:2.0:1789760780.789950:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760780.789967:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5593, 5593] 00020000:00200000:2.0:1789760780.789975:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22908928 chunk: [22908928, 22913024) 22913024, 4096 00020000:00200000:2.0:1789760780.789979:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760780.789997:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5593, 5593] 00000008:00010000:2.0:1789760780.790014:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760780.790020:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760780.790031:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22908928, 22913024) append 0 append lockpos 0 00000080:00200000:2.0:1789760780.790046:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22908928=0x15d9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760780.790103:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760780.790106:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22913024 00000020:00010000:2.0:1789760780.790116:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760780.790119:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.790123:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760780.790126:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760780.790129:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760780.790137:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760780.791521:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760780.791559:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760780.792085:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 22908928<>22913024 00000080:00200000:1.0:1789760780.840109:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5593, 5593] 00000080:00200000:1.0:1789760780.840238:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760780.840246:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760780.840263:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22913024 i_blocks 45063 00000020:00200000:1.0:1789760780.840268:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760780.840279:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760780.840314:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22913024, count: 4096 00000080:00200000:1.0:1789760780.840322:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22913024, bytes: 4096 00000080:00200000:1.0:1789760780.840333:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760780.840341:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760780.840343:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22913024, 22917120) 0 0 00000080:00200000:1.0:1789760780.840348:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760780.840366:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5594, 5594] 00020000:00200000:1.0:1789760780.840409:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22913024 chunk: [22913024, 22917120) 22917120, 4096 00020000:00200000:1.0:1789760780.840414:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760780.840436:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5594, 5594] 00000008:00010000:1.0:1789760780.840454:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760780.840460:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760780.840471:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22913024, 22917120) append 0 append lockpos 0 00000080:00200000:1.0:1789760780.840485:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22913024=0x15da000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760780.840543:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760780.840546:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22917120 00000020:00010000:1.0:1789760780.840554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760780.840559:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.840562:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.840566:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760780.840568:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760780.840577:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760780.840585:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760780.840603:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760780.840776:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22913024<>22917120 00000100:00100000:1.0:1789760780.866804:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972914 gen 3 00000008:00010000:1.0:1789760780.866866:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760780.866874:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.866884:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.866914:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760780.866945:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507351936:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760780.925899:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5595, 5595] 00000080:00200000:1.0:1789760780.925922:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760780.925931:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760780.925947:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22921216 i_blocks 45063 00000020:00200000:1.0:1789760780.925952:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760780.925965:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760780.926000:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22921216, count: 4096 00000080:00200000:1.0:1789760780.926007:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22921216, bytes: 4096 00000080:00200000:1.0:1789760780.926031:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760780.926040:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760780.926043:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22921216, 22925312) 0 0 00000080:00200000:1.0:1789760780.926047:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760780.926065:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5596, 5596] 00020000:00200000:1.0:1789760780.926074:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22921216 chunk: [22921216, 22925312) 22925312, 4096 00020000:00200000:1.0:1789760780.926078:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760780.926100:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5596, 5596] 00000008:00010000:1.0:1789760780.926120:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760780.926126:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760780.926137:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22921216, 22925312) append 0 append lockpos 0 00000080:00200000:1.0:1789760780.926151:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22921216=0x15dc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760780.926445:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760780.926449:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22925312 00000020:00010000:1.0:1789760780.926459:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760780.926462:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.926466:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760780.926469:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760780.926472:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760780.926480:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760780.926487:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760780.926508:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760780.926610:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22921216<>22925312 00000100:00100000:1.0:1789760780.926650:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760780.927147:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507352192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.989301:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972916 gen 3 00000008:00010000:1.0:1789760780.989352:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760780.989360:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760780.989371:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760780.989698:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760780.989739:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507352192:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760780.993913:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507352320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.033285:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972917 gen 3 00000008:00010000:1.0:1789760781.033335:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760781.033344:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.033357:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.033417:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760781.033448:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507352320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.034361:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352448:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760781.065571:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760781.065587:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.065601:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.065623:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760781.065653:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352448:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.067052:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760781.067075:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.098594:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972918 gen 3 00000100:00100000:1.0:1789760781.098639:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25bbb80 x1876699507352576/t4294972919(4294972919) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760797 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760781.098701:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760781.098708:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.098720:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.098807:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760781.098832:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352576:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.103977:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352704:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.147891:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972919 gen 3 00000100:00100000:1.0:1789760781.147938:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25bb480 x1876699507352704/t4294972920(4294972920) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760797 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760781.147991:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760781.147998:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.148010:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.148042:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760781.148349:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352704:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.149997:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760781.150024:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352832:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.225937:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972921 gen 3 00000008:00010000:1.0:1789760781.226005:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760781.226014:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.226026:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.226271:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760781.226317:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352832:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.228001:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352960:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.283222:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972922 gen 3 00000008:00010000:1.0:1789760781.283278:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000020:00200000:1.0:1789760781.283287:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.283300:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.283407:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef79c0 00000100:00100000:1.0:1789760781.283441:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507352960:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.284892:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353088:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.323329:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972923 gen 3 00000008:00010000:1.0:1789760781.323417:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000020:00200000:1.0:1789760781.323427:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.323441:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.323463:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7e40 00000100:00100000:1.0:1789760781.323495:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353088:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.324543:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760781.324566:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353216:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.356270:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972924 gen 3 00000008:00010000:1.0:1789760781.356327:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7240 00000020:00200000:1.0:1789760781.356336:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.356351:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.356455:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7240 00000100:00100000:1.0:1789760781.356488:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353216:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.357669:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353344:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.392643:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972925 gen 3 00000008:00010000:1.0:1789760781.392703:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7900 00000020:00200000:1.0:1789760781.392712:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.392731:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.392794:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7900 00000100:00100000:1.0:1789760781.392824:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353344:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760781.393600:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22962176, 22966272) append 0 append lockpos 0 00000080:00200000:2.0:1789760781.393617:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22962176=0x15e6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760781.393685:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760781.393697:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22966272 00000020:00010000:2.0:1789760781.393707:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760781.393711:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.393715:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c4f0c030 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.393718:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760781.393782:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c4f0c030 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760781.393793:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760781.393801:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760781.393818:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760781.393993:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 22962176<>22966272 00000100:00100000:1.0:1789760781.431651:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507353600:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.465623:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972927 gen 3 00000008:00010000:1.0:1789760781.465680:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000020:00200000:1.0:1789760781.465699:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.465711:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.465737:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7b40 00000100:00100000:1.0:1789760781.465767:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507353600:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760781.466498:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000008:00100000:1.0:1789760781.466601:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22970368<>22974464 00000100:00100000:1.0:1789760781.466647:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353728:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.502747:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972928 gen 3 00000008:00010000:1.0:1789760781.502798:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000020:00200000:1.0:1789760781.502806:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.502819:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.502879:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef70c0 00000100:00100000:1.0:1789760781.502918:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353728:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760781.503546:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760781.503553:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760781.503563:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22974464, 22978560) append 0 append lockpos 0 00000080:00200000:0.0:1789760781.503577:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22974464=0x15e9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760781.503636:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760781.503639:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22978560 00000020:00010000:0.0:1789760781.503647:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760781.503651:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.503654:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.503657:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760781.503659:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760781.503667:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760781.503673:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760781.503706:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000008:00100000:0.0:1789760781.503876:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8e00 - object 0x280000401:3 offset 22974464<>22978560 00000100:00100000:1.0:1789760781.504039:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507353856:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.539456:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972929 gen 3 00000008:00010000:1.0:1789760781.539510:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760781.539520:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.539534:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.539554:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760781.539583:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507353856:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760781.539707:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5609, 5609] 00000080:00200000:0.0:1789760781.539731:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760781.539737:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760781.539754:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22978560 i_blocks 45063 00000020:00200000:0.0:1789760781.539758:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760781.539768:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760781.539801:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22978560, count: 4096 00000080:00200000:0.0:1789760781.539808:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22978560, bytes: 4096 00000080:00200000:0.0:1789760781.539817:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760781.539826:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760781.539829:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22978560, 22982656) 0 0 00000080:00200000:0.0:1789760781.539833:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760781.539849:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5610, 5610] 00020000:00200000:0.0:1789760781.539857:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22978560 chunk: [22978560, 22982656) 22982656, 4096 00020000:00200000:0.0:1789760781.539862:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760781.539881:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5610, 5610] 00000008:00010000:0.0:1789760781.539908:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760781.539916:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760781.539928:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22978560, 22982656) append 0 append lockpos 0 00000080:00200000:0.0:1789760781.539943:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22978560=0x15ea000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760781.539993:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760781.539996:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22982656 00000020:00010000:0.0:1789760781.540004:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760781.540007:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.540010:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.540014:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760781.540017:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760781.540024:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760781.540030:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760781.540045:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7840 00000008:00100000:0.0:1789760781.540147:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bad80 - object 0x280000401:3 offset 22978560<>22982656 00000100:00100000:1.0:1789760781.540295:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353984:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.582410:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972930 gen 3 00000008:00010000:1.0:1789760781.582461:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760781.582470:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.582481:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.582548:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760781.582576:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507353984:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760781.582833:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5610, 5610] 00000080:00200000:0.0:1789760781.582851:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760781.582857:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760781.582872:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22982656 i_blocks 45063 00000020:00200000:0.0:1789760781.582876:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760781.582886:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760781.582965:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22982656, count: 4096 00000080:00200000:0.0:1789760781.582973:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22982656, bytes: 4096 00000080:00200000:0.0:1789760781.582982:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760781.582991:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760781.582994:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22982656, 22986752) 0 0 00000080:00200000:0.0:1789760781.582998:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760781.583014:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5611, 5611] 00020000:00200000:0.0:1789760781.583022:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22982656 chunk: [22982656, 22986752) 22986752, 4096 00020000:00200000:0.0:1789760781.583026:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760781.583044:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5611, 5611] 00000008:00010000:0.0:1789760781.583062:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760781.583068:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760781.583080:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22982656, 22986752) append 0 append lockpos 0 00000080:00200000:0.0:1789760781.583093:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22982656=0x15eb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760781.583213:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760781.583216:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22986752 00000020:00010000:0.0:1789760781.583224:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760781.583228:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.583231:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.583235:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760781.583238:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760781.583247:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760781.583253:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760781.583272:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7600 00000008:00100000:0.0:1789760781.583372:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8700 - object 0x280000401:3 offset 22982656<>22986752 00000100:00100000:1.0:1789760781.583557:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507354112:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760781.627254:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760781.627273:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.627287:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.627319:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760781.627357:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507354112:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760781.627610:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5611, 5611] 00000080:00200000:2.0:1789760781.627630:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760781.627637:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760781.627653:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22986752 i_blocks 45063 00000020:00200000:2.0:1789760781.627657:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760781.627668:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760781.627714:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22986752, count: 4096 00000080:00200000:2.0:1789760781.627789:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22986752, bytes: 4096 00000080:00200000:2.0:1789760781.627801:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760781.627810:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760781.627813:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22986752, 22990848) 0 0 00000080:00200000:2.0:1789760781.627817:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760781.627834:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5612, 5612] 00020000:00200000:2.0:1789760781.627844:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22986752 chunk: [22986752, 22990848) 22990848, 4096 00020000:00200000:2.0:1789760781.627849:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760781.627870:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5612, 5612] 00000008:00010000:2.0:1789760781.627888:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760781.627894:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760781.627978:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22986752, 22990848) append 0 append lockpos 0 00000080:00200000:2.0:1789760781.627992:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22986752=0x15ec000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760781.628053:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760781.628056:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22990848 00000020:00010000:2.0:1789760781.628064:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760781.628067:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.628070:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.628073:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760781.628076:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760781.628083:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760781.628090:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760781.628109:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760781.628214:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 22986752<>22990848 00000100:00100000:2.0:1789760781.628442:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507354240:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760781.656859:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972932 gen 3 00000008:00010000:2.0:1789760781.656926:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760781.656934:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760781.656945:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760781.657130:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760781.657165:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507354240:192.168.203.141@tcp:4:dd.0 00000080:00200008:2.0:1789760781.659845:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22990848, count: 4096 00000080:00200000:2.0:1789760781.659864:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22990848, bytes: 4096 00000080:00200000:2.0:1789760781.659879:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760781.659887:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760781.659889:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22990848, 22994944) 0 0 00000080:00200000:2.0:1789760781.659898:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760781.659918:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5613, 5613] 00020000:00200000:2.0:1789760781.659929:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22990848 chunk: [22990848, 22994944) 22994944, 4096 00020000:00200000:2.0:1789760781.659933:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760781.660004:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5613, 5613] 00000008:00010000:2.0:1789760781.660023:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760781.660031:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760781.660043:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22990848, 22994944) append 0 append lockpos 0 00000080:00200000:2.0:1789760781.660059:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22990848=0x15ed000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760781.660128:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760781.660132:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22994944 00000020:00010000:2.0:1789760781.660141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760781.660146:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.660149:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.660153:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760781.660157:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760781.660168:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760781.660177:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760781.660223:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760781.660339:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 22990848<>22994944 00000080:00200000:2.0:1789760781.702884:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5613, 5613] 00000080:00200000:2.0:1789760781.702901:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760781.702907:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760781.702923:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22994944 i_blocks 45063 00000020:00200000:2.0:1789760781.702927:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760781.702937:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760781.702971:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22994944, count: 4096 00000080:00200000:2.0:1789760781.702978:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22994944, bytes: 4096 00000080:00200000:2.0:1789760781.702988:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760781.702997:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760781.703000:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22994944, 22999040) 0 0 00000080:00200000:2.0:1789760781.703004:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760781.703020:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5614, 5614] 00020000:00200000:2.0:1789760781.703029:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22994944 chunk: [22994944, 22999040) 22999040, 4096 00020000:00200000:2.0:1789760781.703034:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760781.703055:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5614, 5614] 00000008:00010000:2.0:1789760781.703073:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760781.703079:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760781.703091:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22994944, 22999040) append 0 append lockpos 0 00000080:00200000:2.0:1789760781.703105:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22994944=0x15ee000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760781.703165:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760781.703168:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 22999040 00000020:00010000:2.0:1789760781.703176:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760781.703179:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.703182:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.703186:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760781.703190:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760781.703198:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760781.703205:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760781.703220:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760781.704046:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 22994944<>22999040 00000080:00200000:1.0:1789760781.734824:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5614, 5614] 00000080:00200000:1.0:1789760781.734842:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760781.734849:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760781.734865:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 22999040 i_blocks 45063 00000020:00200000:1.0:1789760781.734870:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760781.734880:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760781.734913:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 22999040, count: 4096 00000080:00200000:1.0:1789760781.734921:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 22999040, bytes: 4096 00000080:00200000:1.0:1789760781.734935:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760781.734944:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760781.734947:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [22999040, 23003136) 0 0 00000080:00200000:1.0:1789760781.734951:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760781.734967:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5615, 5615] 00020000:00200000:1.0:1789760781.734975:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 22999040 chunk: [22999040, 23003136) 23003136, 4096 00020000:00200000:1.0:1789760781.734980:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760781.734999:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5615, 5615] 00000008:00010000:1.0:1789760781.735015:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760781.735021:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760781.735032:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [22999040, 23003136) append 0 append lockpos 0 00000080:00200000:1.0:1789760781.735057:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=22999040=0x15ef000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760781.735115:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760781.735118:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23003136 00000020:00010000:1.0:1789760781.735128:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760781.735131:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760781.735134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760781.735138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9a88 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760781.735141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760781.735150:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760781.735157:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760781.735176:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760781.735287:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 22999040<>23003136 00000100:00100000:1.0:1789760781.735339:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760781.735362:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507354624:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.761894:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972935 gen 3 00000008:00010000:1.0:1789760781.761948:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760781.761957:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.761969:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.762043:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760781.762073:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507354624:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760781.763524:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507354752:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760781.791841:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c000 00000020:00200000:2.0:1789760781.791854:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760781.791865:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760781.791878:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c000 00000100:00100000:2.0:1789760781.791902:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507354752:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760781.792601:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507354880:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760781.832740:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972937 gen 3 00000008:00010000:2.0:1789760781.832816:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c180 00000020:00200000:2.0:1789760781.832827:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760781.832839:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760781.832868:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c180 00000100:00100000:2.0:1789760781.832905:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507354880:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760781.834091:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507355008:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760781.865939:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972938 gen 3 00000008:00010000:2.0:1789760781.865992:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5cf00 00000020:00200000:2.0:1789760781.866001:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760781.866012:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760781.866037:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5cf00 00000100:00100000:2.0:1789760781.866075:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507355008:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760781.866358:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5618, 5618] 00000080:00200000:0.0:1789760781.866400:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760781.866408:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760781.866422:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23015424 i_blocks 45063 00000020:00200000:0.0:1789760781.866426:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760781.866435:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760781.866473:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23015424, count: 4096 00000080:00200000:0.0:1789760781.866481:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23015424, bytes: 4096 00000080:00200000:0.0:1789760781.866493:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760781.866502:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760781.866504:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23015424, 23019520) 0 0 00000080:00200000:0.0:1789760781.866509:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760781.866525:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5619, 5619] 00020000:00200000:0.0:1789760781.866534:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23015424 chunk: [23015424, 23019520) 23019520, 4096 00020000:00200000:0.0:1789760781.866538:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760781.866558:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5619, 5619] 00000008:00010000:0.0:1789760781.866575:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760781.866582:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760781.866593:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23015424, 23019520) append 0 append lockpos 0 00000080:00200000:0.0:1789760781.866608:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23015424=0x15f3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760781.866667:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760781.866670:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23019520 00000020:00010000:0.0:1789760781.866679:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760781.866696:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.866700:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760781.866704:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760781.866707:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760781.866716:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760781.866723:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760781.866741:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7540 00000008:00100000:0.0:1789760781.866847:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb100 - object 0x280000401:3 offset 23015424<>23019520 00000100:00100000:0.0:1789760781.866901:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760781.866920:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355136:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760781.902517:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972939 gen 3 00000008:00010000:1.0:1789760781.902567:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760781.902576:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760781.902589:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760781.902621:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760781.902654:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355136:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760781.903148:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5619, 5619] 00000080:00200000:2.0:1789760781.903945:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760781.903956:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760781.903973:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23019520 i_blocks 45063 00000020:00200000:2.0:1789760781.903978:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760781.903991:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760781.904024:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23019520, count: 4096 00000080:00200000:2.0:1789760781.904031:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23019520, bytes: 4096 00000080:00200000:2.0:1789760781.904044:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760781.904053:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760781.904056:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23019520, 23023616) 0 0 00000080:00200000:2.0:1789760781.904060:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760781.904075:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5620, 5620] 00020000:00200000:2.0:1789760781.904082:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23019520 chunk: [23019520, 23023616) 23023616, 4096 00020000:00200000:2.0:1789760781.904086:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760781.904105:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5620, 5620] 00000008:00010000:2.0:1789760781.904122:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760781.904127:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760781.904138:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23019520, 23023616) append 0 append lockpos 0 00000080:00200000:2.0:1789760781.904151:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23019520=0x15f4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760781.904668:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760781.904671:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23023616 00000020:00010000:2.0:1789760781.904692:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760781.904697:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.904700:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760781.904703:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760781.904706:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760781.904715:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760781.904723:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760781.904749:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760781.904860:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 23019520<>23023616 00000100:00100000:2.0:1789760781.904917:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760781.905060:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507355264:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760781.944896:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972940 gen 3 00000008:00010000:2.0:1789760781.945006:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:2.0:1789760781.945016:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760781.945029:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760781.945058:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:2.0:1789760781.945095:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507355264:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760781.945498:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5620, 5620] 00000080:00200000:1.0:1789760781.945518:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760781.945525:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760781.945541:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23023616 i_blocks 45063 00000020:00200000:1.0:1789760781.945545:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760781.945555:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760781.945591:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23023616, count: 4096 00000080:00200000:1.0:1789760781.945600:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23023616, bytes: 4096 00000080:00200000:1.0:1789760781.945611:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760781.945620:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760781.945622:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23023616, 23027712) 0 0 00000080:00200000:1.0:1789760781.945627:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760781.949027:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5621, 5621] 00020000:00200000:1.0:1789760781.949045:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23023616 chunk: [23023616, 23027712) 23027712, 4096 00020000:00200000:1.0:1789760781.949053:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760781.949081:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5621, 5621] 00000008:00010000:1.0:1789760781.949104:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760781.949110:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760781.949136:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23023616, 23027712) append 0 append lockpos 0 00000080:00200000:1.0:1789760781.949152:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23023616=0x15f5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760781.949448:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760781.949453:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23027712 00000020:00010000:1.0:1789760781.949465:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760781.949468:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760781.949472:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760781.949475:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760781.949478:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760781.949489:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760781.949499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760781.949528:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760781.949652:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23023616<>23027712 00000100:00100000:1.0:1789760781.949709:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760781.949737:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507355392:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760782.000053:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760782.000069:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.000080:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.000135:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760782.000172:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507355392:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.004591:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5621, 5621] 00000080:00200000:0.0:1789760782.004624:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.004631:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.004890:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23027712 i_blocks 45063 00000020:00200000:0.0:1789760782.004898:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.004919:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.004957:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23027712, count: 4096 00000080:00200000:0.0:1789760782.004966:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23027712, bytes: 4096 00000080:00200000:0.0:1789760782.004981:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.004991:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.004993:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23027712, 23031808) 0 0 00000080:00200000:0.0:1789760782.004998:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.005015:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5622, 5622] 00020000:00200000:0.0:1789760782.005025:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23027712 chunk: [23027712, 23031808) 23031808, 4096 00020000:00200000:0.0:1789760782.005030:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.005051:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5622, 5622] 00000008:00010000:0.0:1789760782.005071:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.005078:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.005090:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23027712, 23031808) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.005109:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23027712=0x15f6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.005183:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.005186:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23031808 00000020:00010000:0.0:1789760782.005195:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.005198:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.005201:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.005204:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.005207:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.005217:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.005224:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.005242:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000008:00100000:0.0:1789760782.005365:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb100 - object 0x280000401:3 offset 23027712<>23031808 00000100:00100000:0.0:1789760782.005447:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760782.005475:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355520:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.067123:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972942 gen 3 00000008:00010000:1.0:1789760782.067205:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760782.067214:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.067226:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.067370:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760782.067435:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355520:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.077811:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5622, 5622] 00000080:00200000:0.0:1789760782.077843:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.077851:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.077873:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23031808 i_blocks 45063 00000020:00200000:0.0:1789760782.077879:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.077898:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.077936:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23031808, count: 4096 00000080:00200000:0.0:1789760782.077946:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23031808, bytes: 4096 00000080:00200000:0.0:1789760782.077969:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.077981:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.077983:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23031808, 23035904) 0 0 00000080:00200000:0.0:1789760782.077989:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.078010:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5623, 5623] 00020000:00200000:0.0:1789760782.078019:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23031808 chunk: [23031808, 23035904) 23035904, 4096 00020000:00200000:0.0:1789760782.078025:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.078049:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5623, 5623] 00000008:00010000:0.0:1789760782.078072:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.078079:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.078093:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23031808, 23035904) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.078108:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23031808=0x15f7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.078190:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.078193:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23035904 00000020:00010000:0.0:1789760782.078202:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.078206:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.078209:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c54177e8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.078213:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.078216:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.078227:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.078234:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.078265:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760782.078417:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8700 - object 0x280000401:3 offset 23031808<>23035904 00000100:00100000:1.0:1789760782.079180:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355648:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760782.128077:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760782.128095:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.128110:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.128219:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:1.0:1789760782.128252:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355648:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.128341:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5623, 5623] 00000080:00200000:0.0:1789760782.128359:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.128365:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760782.128471:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23035904 i_blocks 45063 00000020:00200000:1.0:1789760782.128476:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760782.128486:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760782.128523:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23035904, count: 4096 00000080:00200000:1.0:1789760782.128531:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23035904, bytes: 4096 00000080:00200000:1.0:1789760782.128542:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760782.128552:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760782.128554:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23035904, 23040000) 0 0 00000080:00200000:1.0:1789760782.128559:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760782.128579:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5624, 5624] 00020000:00200000:1.0:1789760782.128588:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23035904 chunk: [23035904, 23040000) 23040000, 4096 00020000:00200000:1.0:1789760782.128593:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760782.128614:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5624, 5624] 00000008:00010000:1.0:1789760782.128632:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760782.128637:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760782.128649:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23035904, 23040000) append 0 append lockpos 0 00000080:00200000:1.0:1789760782.128663:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23035904=0x15f8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760782.128833:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760782.128836:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23040000 00000020:00010000:1.0:1789760782.128846:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760782.128850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760782.128853:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760782.128858:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760782.128861:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760782.128869:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760782.128876:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760782.128892:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760782.129111:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23035904<>23040000 00000100:00100000:0.0:1789760782.129249:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507355776:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.166582:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972944 gen 3 00000008:00010000:1.0:1789760782.166650:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760782.166659:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.166668:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.166701:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760782.166728:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507355776:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760782.166783:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5624, 5624] 00000080:00200000:2.0:1789760782.166803:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.166809:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.166824:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23040000 i_blocks 45063 00000020:00200000:2.0:1789760782.166828:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.166849:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.166883:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23040000, count: 4096 00000080:00200000:2.0:1789760782.166891:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23040000, bytes: 4096 00000080:00200000:2.0:1789760782.166902:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.166911:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.166923:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23040000, 23044096) 0 0 00000080:00200000:2.0:1789760782.166928:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.166955:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5625, 5625] 00020000:00200000:2.0:1789760782.166965:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23040000 chunk: [23040000, 23044096) 23044096, 4096 00020000:00200000:2.0:1789760782.166969:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.166991:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5625, 5625] 00000008:00010000:2.0:1789760782.167008:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.167014:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.167026:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23040000, 23044096) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.167041:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23040000=0x15f9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.167092:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.167095:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23044096 00000020:00010000:2.0:1789760782.167104:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.167108:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.167290:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.167294:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.167298:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.167306:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.167314:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760782.167455:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:1.0:1789760782.167556:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23040000<>23044096 00000100:00100000:1.0:1789760782.167599:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355904:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.218285:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972945 gen 3 00000008:00010000:1.0:1789760782.218339:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:1.0:1789760782.218348:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.218360:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.218551:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:1.0:1789760782.218583:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507355904:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760782.219881:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5625, 5625] 00000080:00200000:2.0:1789760782.220318:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.220326:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.220344:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23044096 i_blocks 45063 00000020:00200000:2.0:1789760782.220359:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.220373:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.220428:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23044096, count: 4096 00000080:00200000:2.0:1789760782.220437:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23044096, bytes: 4096 00000080:00200000:2.0:1789760782.220452:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.220460:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.220463:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23044096, 23048192) 0 0 00000080:00200000:2.0:1789760782.220467:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.220483:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5626, 5626] 00020000:00200000:2.0:1789760782.220491:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23044096 chunk: [23044096, 23048192) 23048192, 4096 00020000:00200000:2.0:1789760782.220496:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.220517:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5626, 5626] 00000008:00010000:2.0:1789760782.220537:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.220543:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.220555:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23044096, 23048192) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.220571:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23044096=0x15fa000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.220643:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.220646:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23048192 00000020:00010000:2.0:1789760782.220655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.220659:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.220662:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.220667:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.220670:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.220690:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.220698:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.220726:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760782.220863:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 23044096<>23048192 00000080:00200000:0.0:1789760782.256764:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5626, 5626] 00000080:00200000:0.0:1789760782.256780:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.256787:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.256802:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23048192 i_blocks 45063 00000020:00200000:0.0:1789760782.256806:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.256816:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.256849:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23048192, count: 4096 00000080:00200000:0.0:1789760782.256856:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23048192, bytes: 4096 00000080:00200000:0.0:1789760782.256866:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.256874:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.256877:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23048192, 23052288) 0 0 00000080:00200000:0.0:1789760782.256881:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.256906:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5627, 5627] 00020000:00200000:0.0:1789760782.256915:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23048192 chunk: [23048192, 23052288) 23052288, 4096 00020000:00200000:0.0:1789760782.256919:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.256937:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5627, 5627] 00000008:00010000:0.0:1789760782.256955:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.256960:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.256972:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23048192, 23052288) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.256985:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23048192=0x15fb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.257039:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.257041:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23052288 00000020:00010000:0.0:1789760782.257049:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.257053:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.257056:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.257059:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.257062:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.257072:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.257078:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.257096:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000008:00100000:0.0:1789760782.257206:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bad80 - object 0x280000401:3 offset 23048192<>23052288 00000100:00100000:1.0:1789760782.257785:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356160:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760782.293001:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:1.0:1789760782.293019:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.293031:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.293052:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:1.0:1789760782.293086:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356160:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.293243:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5627, 5627] 00000080:00200000:0.0:1789760782.293258:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.293264:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.293279:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23052288 i_blocks 45063 00000020:00200000:0.0:1789760782.293283:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.293292:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.293332:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23052288, count: 4096 00000080:00200000:0.0:1789760782.293339:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23052288, bytes: 4096 00000080:00200000:0.0:1789760782.293348:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.293357:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.293359:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23052288, 23056384) 0 0 00000080:00200000:0.0:1789760782.293363:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.293407:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5628, 5628] 00020000:00200000:0.0:1789760782.293519:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23052288 chunk: [23052288, 23056384) 23056384, 4096 00020000:00200000:0.0:1789760782.293524:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.293543:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5628, 5628] 00000008:00010000:0.0:1789760782.293559:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.293566:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.293577:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23052288, 23056384) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.293589:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23052288=0x15fc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.293647:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.293650:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23056384 00000020:00010000:0.0:1789760782.293660:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.293664:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.293668:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.293672:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.293676:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.293699:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.293707:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.293725:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7480 00000008:00100000:0.0:1789760782.293843:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8e00 - object 0x280000401:3 offset 23052288<>23056384 00000100:00100000:1.0:1789760782.294116:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507356288:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.324025:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972948 gen 3 00000008:00010000:1.0:1789760782.324088:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:1.0:1789760782.324096:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.324108:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.324129:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000080:00200000:0.0:1789760782.324157:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5628, 5628] 00000100:00100000:1.0:1789760782.324162:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507356288:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.324241:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.324248:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.324265:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23056384 i_blocks 45063 00000020:00200000:0.0:1789760782.324270:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.324282:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.324316:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23056384, count: 4096 00000080:00200000:0.0:1789760782.324323:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23056384, bytes: 4096 00000080:00200000:0.0:1789760782.324332:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.324341:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.324344:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23056384, 23060480) 0 0 00000080:00200000:0.0:1789760782.324349:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.324365:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5629, 5629] 00020000:00200000:0.0:1789760782.324374:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23056384 chunk: [23056384, 23060480) 23060480, 4096 00020000:00200000:0.0:1789760782.324413:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.324434:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5629, 5629] 00000008:00010000:0.0:1789760782.324738:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.324745:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.324757:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23056384, 23060480) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.324771:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23056384=0x15fd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.324828:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.324832:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23060480 00000020:00010000:0.0:1789760782.324841:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.324845:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.324848:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.324851:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.324854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.324864:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.326374:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.326432:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000008:00100000:0.0:1789760782.326567:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9f80 - object 0x280000401:3 offset 23056384<>23060480 00000100:00100000:1.0:1789760782.329116:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356416:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.359037:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972949 gen 3 00000008:00010000:1.0:1789760782.359090:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760782.359099:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.359113:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.359163:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760782.359191:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356416:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.359266:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5629, 5629] 00000080:00200000:0.0:1789760782.359287:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.359295:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.359313:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23060480 i_blocks 45063 00000020:00200000:0.0:1789760782.359317:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.359330:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.359369:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23060480, count: 4096 00000080:00200000:0.0:1789760782.359405:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23060480, bytes: 4096 00000080:00200000:0.0:1789760782.359416:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.359425:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.359428:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23060480, 23064576) 0 0 00000080:00200000:0.0:1789760782.359433:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.359453:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5630, 5630] 00020000:00200000:0.0:1789760782.359463:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23060480 chunk: [23060480, 23064576) 23064576, 4096 00020000:00200000:0.0:1789760782.359467:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.359491:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5630, 5630] 00000008:00010000:0.0:1789760782.359508:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.359514:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.359526:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23060480, 23064576) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.359542:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23060480=0x15fe000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.359604:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.359612:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23064576 00000020:00010000:0.0:1789760782.359621:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.359625:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.359628:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.359632:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.359635:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.359645:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.359652:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.359667:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7780 00000008:00100000:0.0:1789760782.359836:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25ba680 - object 0x280000401:3 offset 23060480<>23064576 00000100:00100000:1.0:1789760782.359894:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507356544:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.391146:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972950 gen 3 00000008:00010000:1.0:1789760782.391182:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760782.391403:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.391415:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.391435:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760782.391459:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507356544:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.391663:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5630, 5630] 00000080:00200000:0.0:1789760782.391676:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.391690:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.391702:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23064576 i_blocks 45063 00000020:00200000:0.0:1789760782.391705:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.391714:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.391888:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23064576, count: 4096 00000080:00200000:0.0:1789760782.391896:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23064576, bytes: 4096 00000080:00200000:0.0:1789760782.391904:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.391911:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.391913:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23064576, 23068672) 0 0 00000080:00200000:0.0:1789760782.391918:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.391930:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5631, 5631] 00020000:00200000:0.0:1789760782.391936:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23064576 chunk: [23064576, 23068672) 23068672, 4096 00020000:00200000:0.0:1789760782.391940:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.391954:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5631, 5631] 00000008:00010000:0.0:1789760782.391967:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.391972:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.391981:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23064576, 23068672) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.391990:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23064576=0x15ff000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.392047:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.392050:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23068672 00000020:00010000:0.0:1789760782.392057:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.392059:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.392062:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.392065:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.392067:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.392074:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.392080:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.392094:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000008:00100000:0.0:1789760782.392179:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9180 - object 0x280000401:3 offset 23064576<>23068672 00000100:00100000:1.0:1789760782.392258:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356672:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.423610:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972951 gen 3 00000008:00010000:1.0:1789760782.423658:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760782.423667:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.423691:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.423711:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000080:00200000:0.0:1789760782.423813:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5631, 5631] 00000080:00200000:0.0:1789760782.423832:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.423838:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000100:00100000:1.0:1789760782.423852:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356672:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.423854:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23068672 i_blocks 45063 00000020:00200000:0.0:1789760782.423857:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.423868:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.423903:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23068672, count: 4096 00000080:00200000:0.0:1789760782.423911:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23068672, bytes: 4096 00000080:00200000:0.0:1789760782.423921:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.423929:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.423931:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23068672, 23072768) 0 0 00000080:00200000:0.0:1789760782.423936:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.423953:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5632, 5632] 00020000:00200000:0.0:1789760782.423961:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23068672 chunk: [23068672, 23072768) 23072768, 4096 00020000:00200000:0.0:1789760782.423965:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.423985:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5632, 5632] 00000008:00010000:0.0:1789760782.424009:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.424015:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.424026:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23068672, 23072768) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.424040:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23068672=0x1600000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.424098:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.424101:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23072768 00000020:00010000:0.0:1789760782.424108:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.424112:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.424115:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.424118:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.424121:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.424189:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.424197:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.424213:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000008:00100000:0.0:1789760782.424311:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8a80 - object 0x280000401:3 offset 23068672<>23072768 00000100:00100000:1.0:1789760782.424416:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507356800:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.461833:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972952 gen 3 00000008:00010000:1.0:1789760782.461882:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000020:00200000:1.0:1789760782.461889:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.461899:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.461967:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7d80 00000100:00100000:1.0:1789760782.462000:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507356800:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.462199:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5632, 5632] 00000080:00200000:0.0:1789760782.462214:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.462221:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.462236:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23072768 i_blocks 45063 00000020:00200000:0.0:1789760782.462240:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.462250:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.462285:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23072768, count: 4096 00000080:00200000:0.0:1789760782.462292:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23072768, bytes: 4096 00000080:00200000:0.0:1789760782.462302:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.462309:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.462312:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23072768, 23076864) 0 0 00000080:00200000:0.0:1789760782.462316:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.462332:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5633, 5633] 00020000:00200000:0.0:1789760782.462340:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23072768 chunk: [23072768, 23076864) 23076864, 4096 00020000:00200000:0.0:1789760782.462344:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.462364:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5633, 5633] 00000008:00010000:0.0:1789760782.462416:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.462423:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.462436:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23072768, 23076864) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.462451:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23072768=0x1601000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.462650:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.462653:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23076864 00000020:00010000:0.0:1789760782.462662:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.462666:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.462669:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.462673:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.462676:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.462694:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.462702:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.462721:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7780 00000008:00100000:0.0:1789760782.462826:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8380 - object 0x280000401:3 offset 23072768<>23076864 00000100:00100000:1.0:1789760782.462896:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356928:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.488575:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972953 gen 3 00000008:00010000:1.0:1789760782.488619:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7780 00000020:00200000:1.0:1789760782.488628:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.488640:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.488710:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7780 00000100:00100000:1.0:1789760782.488741:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507356928:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760782.488861:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5633, 5633] 00000080:00200000:0.0:1789760782.488879:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760782.488884:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760782.488899:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23076864 i_blocks 47111 00000020:00200000:0.0:1789760782.488903:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760782.488914:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760782.488948:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23076864, count: 4096 00000080:00200000:0.0:1789760782.488955:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23076864, bytes: 4096 00000080:00200000:0.0:1789760782.488966:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760782.488975:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760782.488978:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23076864, 23080960) 0 0 00000080:00200000:0.0:1789760782.488982:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760782.488998:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5634, 5634] 00020000:00200000:0.0:1789760782.489007:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23076864 chunk: [23076864, 23080960) 23080960, 4096 00020000:00200000:0.0:1789760782.489011:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760782.489116:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5634, 5634] 00000008:00010000:0.0:1789760782.489191:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760782.489198:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760782.489212:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23076864, 23080960) append 0 append lockpos 0 00000080:00200000:0.0:1789760782.489226:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23076864=0x1602000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760782.489283:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760782.489287:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23080960 00000020:00010000:0.0:1789760782.489295:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760782.489299:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.489302:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760782.489306:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760782.489309:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760782.489318:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760782.489325:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760782.489342:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000008:00100000:0.0:1789760782.489566:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb800 - object 0x280000401:3 offset 23076864<>23080960 00000100:00100000:1.0:1789760782.489616:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760782.489693:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507357056:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.514241:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972954 gen 3 00000008:00010000:1.0:1789760782.514291:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000020:00200000:1.0:1789760782.514301:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.514312:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.514340:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7a80 00000100:00100000:1.0:1789760782.514372:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507357056:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760782.514536:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5634, 5634] 00000080:00200000:2.0:1789760782.514552:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.514558:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.514572:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23080960 i_blocks 47111 00000020:00200000:2.0:1789760782.514575:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.514584:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.514614:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23080960, count: 4096 00000080:00200000:2.0:1789760782.514620:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23080960, bytes: 4096 00000080:00200000:2.0:1789760782.514627:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.514634:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.514636:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23080960, 23085056) 0 0 00000080:00200000:2.0:1789760782.514640:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.514654:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5635, 5635] 00020000:00200000:2.0:1789760782.514662:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23080960 chunk: [23080960, 23085056) 23085056, 4096 00020000:00200000:2.0:1789760782.514666:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.514693:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5635, 5635] 00000008:00010000:2.0:1789760782.514715:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.514721:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.514733:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23080960, 23085056) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.514748:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23080960=0x1603000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.514804:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.514807:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23085056 00000020:00010000:2.0:1789760782.514814:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.514818:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.514821:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.514826:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.514833:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.514840:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.514847:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.514863:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760782.514966:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 23080960<>23085056 00000008:00010000:2.0:1789760782.550916:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000020:00200000:2.0:1789760782.550926:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760782.550938:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760782.551016:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000100:00100000:2.0:1789760782.551047:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507357184:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760782.551623:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5635, 5635] 00000080:00200000:1.0:1789760782.551642:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760782.551649:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760782.551666:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23085056 i_blocks 47111 00000020:00200000:1.0:1789760782.551671:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760782.551701:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760782.551735:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23085056, count: 4096 00000080:00200000:1.0:1789760782.551752:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23085056, bytes: 4096 00000080:00200000:1.0:1789760782.551774:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760782.551783:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760782.551785:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23085056, 23089152) 0 0 00000080:00200000:1.0:1789760782.551790:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760782.551806:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5636, 5636] 00020000:00200000:1.0:1789760782.551815:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23085056 chunk: [23085056, 23089152) 23089152, 4096 00020000:00200000:1.0:1789760782.551820:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760782.551841:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5636, 5636] 00000008:00010000:1.0:1789760782.551860:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760782.551866:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760782.551878:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23085056, 23089152) append 0 append lockpos 0 00000080:00200000:1.0:1789760782.551893:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23085056=0x1604000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760782.551960:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760782.551963:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23089152 00000020:00010000:1.0:1789760782.551973:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760782.551977:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760782.551980:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760782.551984:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760782.551987:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760782.551996:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760782.552004:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760782.552028:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760782.552150:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23085056<>23089152 00000100:00100000:0.0:1789760782.552267:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507357312:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.581907:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972956 gen 3 00000008:00010000:1.0:1789760782.581959:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760782.581968:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.581979:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760782.582004:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760782.582929:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507357312:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760782.584533:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5636, 5636] 00000080:00200000:2.0:1789760782.584559:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.584566:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.584585:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23089152 i_blocks 47111 00000020:00200000:2.0:1789760782.584591:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.584607:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.584644:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23089152, count: 4096 00000080:00200000:2.0:1789760782.584652:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23089152, bytes: 4096 00000080:00200000:2.0:1789760782.584667:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.584688:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.584690:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23089152, 23093248) 0 0 00000080:00200000:2.0:1789760782.584694:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.584714:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5637, 5637] 00020000:00200000:2.0:1789760782.584725:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23089152 chunk: [23089152, 23093248) 23093248, 4096 00020000:00200000:2.0:1789760782.584729:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.584754:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5637, 5637] 00000008:00010000:2.0:1789760782.584777:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.584784:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.584798:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23089152, 23093248) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.584814:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23089152=0x1605000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.584891:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.584894:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23093248 00000020:00010000:2.0:1789760782.584903:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.584907:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.584910:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.584913:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.584916:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.584925:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.584932:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.584957:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760782.585076:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 23089152<>23093248 00000080:00200000:2.0:1789760782.632633:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5637, 5637] 00000080:00200000:2.0:1789760782.632650:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.632657:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.632674:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23093248 i_blocks 47111 00000020:00200000:2.0:1789760782.632690:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.632701:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.632736:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23093248, count: 4096 00000080:00200000:2.0:1789760782.632744:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23093248, bytes: 4096 00000080:00200000:2.0:1789760782.632753:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.632762:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.632765:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23093248, 23097344) 0 0 00000080:00200000:2.0:1789760782.632770:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.632787:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5638, 5638] 00020000:00200000:2.0:1789760782.632795:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23093248 chunk: [23093248, 23097344) 23097344, 4096 00020000:00200000:2.0:1789760782.632799:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.632819:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5638, 5638] 00000008:00010000:2.0:1789760782.632836:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.632842:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.632853:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23093248, 23097344) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.632867:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23093248=0x1606000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.632926:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.632929:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23097344 00000020:00010000:2.0:1789760782.632938:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.632941:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.632944:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.632947:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.632950:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.632959:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.632966:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.632982:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760782.633086:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 23093248<>23097344 00000080:00200000:2.0:1789760782.684224:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5638, 5638] 00000080:00200000:2.0:1789760782.684241:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.684248:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.684263:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23097344 i_blocks 47111 00000020:00200000:2.0:1789760782.684266:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.684276:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.684309:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23097344, count: 4096 00000080:00200000:2.0:1789760782.684316:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23097344, bytes: 4096 00000080:00200000:2.0:1789760782.684324:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.684332:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.684334:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23097344, 23101440) 0 0 00000080:00200000:2.0:1789760782.684339:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.684355:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5639, 5639] 00020000:00200000:2.0:1789760782.684363:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23097344 chunk: [23097344, 23101440) 23101440, 4096 00020000:00200000:2.0:1789760782.684367:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.684435:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5639, 5639] 00000008:00010000:2.0:1789760782.684452:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.684457:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.684468:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23097344, 23101440) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.684482:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23097344=0x1607000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.684536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.684539:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23101440 00000020:00010000:2.0:1789760782.684548:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.684551:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.684554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.684557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.684560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.684567:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.684574:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.684590:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760782.684706:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 23097344<>23101440 00000080:00200000:1.0:1789760782.736514:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5639, 5639] 00000080:00200000:1.0:1789760782.736534:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760782.736541:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760782.736558:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23101440 i_blocks 47111 00000020:00200000:1.0:1789760782.736563:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760782.736577:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760782.736614:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23101440, count: 4096 00000080:00200000:1.0:1789760782.736622:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23101440, bytes: 4096 00000080:00200000:1.0:1789760782.736636:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760782.736647:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760782.736649:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23101440, 23105536) 0 0 00000080:00200000:1.0:1789760782.736653:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760782.736673:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5640, 5640] 00020000:00200000:1.0:1789760782.736691:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23101440 chunk: [23101440, 23105536) 23105536, 4096 00020000:00200000:1.0:1789760782.736697:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760782.736718:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5640, 5640] 00000008:00010000:1.0:1789760782.736742:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760782.736749:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760782.736761:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23101440, 23105536) append 0 append lockpos 0 00000080:00200000:1.0:1789760782.736777:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23101440=0x1608000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760782.736844:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760782.736847:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23105536 00000020:00010000:1.0:1789760782.736855:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760782.736860:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760782.736862:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760782.736866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760782.736869:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760782.736877:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760782.736885:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760782.736907:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760782.737020:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23101440<>23105536 00000100:00100000:1.0:1789760782.737075:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760782.737099:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507357824:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760782.795443:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972960 gen 3 00000008:00010000:1.0:1789760782.795497:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760782.795507:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760782.795519:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.795598:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5640, 5640] 00000080:00200000:2.0:1789760782.795621:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.795628:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.795643:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23105536 i_blocks 47111 00000020:00200000:2.0:1789760782.795647:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.795658:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000008:00010000:1.0:1789760782.795685:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000080:00200008:2.0:1789760782.795706:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23105536, count: 4096 00000080:00200000:2.0:1789760782.795715:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23105536, bytes: 4096 00000080:00200000:2.0:1789760782.795729:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.795740:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.795742:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23105536, 23109632) 0 0 00000080:00200000:2.0:1789760782.795747:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.795762:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5641, 5641] 00020000:00200000:2.0:1789760782.795771:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23105536 chunk: [23105536, 23109632) 23109632, 4096 00020000:00200000:2.0:1789760782.795775:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.795795:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5641, 5641] 00000008:00010000:2.0:1789760782.795814:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.795821:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.795833:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23105536, 23109632) append 0 append lockpos 0 00000100:00100000:1.0:1789760782.795836:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507357824:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760782.795849:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23105536=0x1609000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.795911:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.795915:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23109632 00000020:00010000:2.0:1789760782.795923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.795927:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.795929:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.795933:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.795935:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.795943:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.795949:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.795965:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760782.796067:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 23105536<>23109632 00000080:00200000:2.0:1789760782.835196:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5641, 5641] 00000080:00200000:2.0:1789760782.835215:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.835222:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.835239:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23109632 i_blocks 47111 00000020:00200000:2.0:1789760782.835244:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.835254:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.835289:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23109632, count: 4096 00000080:00200000:2.0:1789760782.835297:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23109632, bytes: 4096 00000080:00200000:2.0:1789760782.835308:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.835317:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.835320:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23109632, 23113728) 0 0 00000080:00200000:2.0:1789760782.835324:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.835341:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5642, 5642] 00020000:00200000:2.0:1789760782.835350:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23109632 chunk: [23109632, 23113728) 23113728, 4096 00020000:00200000:2.0:1789760782.835355:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.835375:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5642, 5642] 00000008:00010000:2.0:1789760782.835428:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.835434:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.835446:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23109632, 23113728) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.835464:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23109632=0x160a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.835525:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.835528:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23113728 00000020:00010000:2.0:1789760782.835538:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.835542:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.835545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.835549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.835552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.835560:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.835568:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.835587:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760782.835705:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 23109632<>23113728 00000080:00200000:2.0:1789760782.877299:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5642, 5642] 00000080:00200000:2.0:1789760782.877315:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.877322:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.877337:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23113728 i_blocks 47111 00000020:00200000:2.0:1789760782.877341:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.877352:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.877430:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23113728, count: 4096 00000080:00200000:2.0:1789760782.877438:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23113728, bytes: 4096 00000080:00200000:2.0:1789760782.877449:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.877458:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.877460:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23113728, 23117824) 0 0 00000080:00200000:2.0:1789760782.877465:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.877482:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5643, 5643] 00020000:00200000:2.0:1789760782.877492:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23113728 chunk: [23113728, 23117824) 23117824, 4096 00020000:00200000:2.0:1789760782.877497:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.877518:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5643, 5643] 00000008:00010000:2.0:1789760782.877537:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.877543:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.877556:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23113728, 23117824) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.877571:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23113728=0x160b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.877631:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.877634:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23117824 00000020:00010000:2.0:1789760782.877643:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.877646:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.877650:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.877654:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.877656:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.877664:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.877671:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.877698:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760782.877819:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 23113728<>23117824 00000080:00200000:2.0:1789760782.927238:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5643, 5643] 00000080:00200000:2.0:1789760782.927255:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.927262:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.927277:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23117824 i_blocks 47111 00000020:00200000:2.0:1789760782.927281:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.927291:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.927325:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23117824, count: 4096 00000080:00200000:2.0:1789760782.927334:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23117824, bytes: 4096 00000080:00200000:2.0:1789760782.927344:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.927354:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.927356:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23117824, 23121920) 0 0 00000080:00200000:2.0:1789760782.927361:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.927429:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5644, 5644] 00020000:00200000:2.0:1789760782.927438:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23117824 chunk: [23117824, 23121920) 23121920, 4096 00020000:00200000:2.0:1789760782.927443:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.927483:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5644, 5644] 00000008:00010000:2.0:1789760782.927503:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.927509:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.927521:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23117824, 23121920) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.927536:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23117824=0x160c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.927594:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.927596:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23121920 00000020:00010000:2.0:1789760782.927604:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.927607:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.927611:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.927614:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.927617:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.927624:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.927631:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.927643:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760782.927833:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 23117824<>23121920 00000080:00200000:2.0:1789760782.969977:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5644, 5644] 00000080:00200000:2.0:1789760782.969994:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760782.970000:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760782.970017:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23121920 i_blocks 47111 00000020:00200000:2.0:1789760782.970020:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760782.970029:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760782.970062:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23121920, count: 4096 00000080:00200000:2.0:1789760782.970069:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23121920, bytes: 4096 00000080:00200000:2.0:1789760782.970079:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760782.970088:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760782.970090:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23121920, 23126016) 0 0 00000080:00200000:2.0:1789760782.970094:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760782.970110:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5645, 5645] 00020000:00200000:2.0:1789760782.970118:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23121920 chunk: [23121920, 23126016) 23126016, 4096 00020000:00200000:2.0:1789760782.970132:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760782.970151:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5645, 5645] 00000008:00010000:2.0:1789760782.970169:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760782.970174:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760782.970186:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23121920, 23126016) append 0 append lockpos 0 00000080:00200000:2.0:1789760782.970199:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23121920=0x160d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760782.970258:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760782.970260:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23126016 00000020:00010000:2.0:1789760782.970269:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760782.970272:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.970275:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760782.970279:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760782.970281:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760782.970288:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760782.970295:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760782.970311:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760782.970462:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 23121920<>23126016 00000080:00200000:2.0:1789760783.003423:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5645, 5645] 00000080:00200000:2.0:1789760783.003441:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.003447:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.003463:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23126016 i_blocks 47111 00000020:00200000:2.0:1789760783.003468:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.003479:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.003513:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23126016, count: 4096 00000080:00200000:2.0:1789760783.003520:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23126016, bytes: 4096 00000080:00200000:2.0:1789760783.003531:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.003540:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.003542:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23126016, 23130112) 0 0 00000080:00200000:2.0:1789760783.003547:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.003564:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5646, 5646] 00020000:00200000:2.0:1789760783.003572:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23126016 chunk: [23126016, 23130112) 23130112, 4096 00020000:00200000:2.0:1789760783.003577:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.003596:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5646, 5646] 00000008:00010000:2.0:1789760783.003612:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.003618:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.003629:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23126016, 23130112) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.003644:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23126016=0x160e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.003709:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.003720:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23130112 00000020:00010000:2.0:1789760783.003728:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.003732:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.003735:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.003738:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.003741:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.003749:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.003756:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.003772:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760783.003905:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 23126016<>23130112 00000080:00200000:2.0:1789760783.045774:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5646, 5646] 00000080:00200000:2.0:1789760783.045791:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.045797:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.045813:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23130112 i_blocks 47111 00000020:00200000:2.0:1789760783.045817:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.045828:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.045861:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23130112, count: 4096 00000080:00200000:2.0:1789760783.045868:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23130112, bytes: 4096 00000080:00200000:2.0:1789760783.045877:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.045886:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.045888:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23130112, 23134208) 0 0 00000080:00200000:2.0:1789760783.045893:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.045922:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5647, 5647] 00020000:00200000:2.0:1789760783.045931:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23130112 chunk: [23130112, 23134208) 23134208, 4096 00020000:00200000:2.0:1789760783.045936:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.045956:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5647, 5647] 00000008:00010000:2.0:1789760783.045983:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.045989:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.046000:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23130112, 23134208) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.046014:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23130112=0x160f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.046071:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.046075:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23134208 00000020:00010000:2.0:1789760783.046083:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.046086:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.046089:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.046092:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.046095:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.046103:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.046110:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.046291:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760783.046440:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 23130112<>23134208 00000080:00200000:2.0:1789760783.083689:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5647, 5647] 00000080:00200000:2.0:1789760783.083707:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.083714:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.083731:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23134208 i_blocks 47111 00000020:00200000:2.0:1789760783.083735:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.083745:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.083780:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23134208, count: 4096 00000080:00200000:2.0:1789760783.083787:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23134208, bytes: 4096 00000080:00200000:2.0:1789760783.083797:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.083808:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.083810:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23134208, 23138304) 0 0 00000080:00200000:2.0:1789760783.083815:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.083832:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5648, 5648] 00020000:00200000:2.0:1789760783.083840:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23134208 chunk: [23134208, 23138304) 23138304, 4096 00020000:00200000:2.0:1789760783.083844:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.083865:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5648, 5648] 00000008:00010000:2.0:1789760783.083884:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.083890:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.083913:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23134208, 23138304) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.083929:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23134208=0x1610000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.084057:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.084061:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23138304 00000020:00010000:2.0:1789760783.084070:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.084074:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.084077:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.084081:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.084084:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.084092:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.084100:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.084179:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760783.084285:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 23134208<>23138304 00000008:00010000:2.0:1789760783.128526:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ace40 00000020:00200000:2.0:1789760783.128545:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760783.128556:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760783.128581:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ace40 00000100:00100000:2.0:1789760783.128616:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507358848:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.128723:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5648, 5648] 00000080:00200000:1.0:1789760783.128739:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.128746:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.128762:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23138304 i_blocks 47111 00000020:00200000:1.0:1789760783.128766:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.128776:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.128813:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23138304, count: 4096 00000080:00200000:1.0:1789760783.128822:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23138304, bytes: 4096 00000080:00200000:1.0:1789760783.128831:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.128840:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.128843:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23138304, 23142400) 0 0 00000080:00200000:1.0:1789760783.128848:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.128864:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5649, 5649] 00020000:00200000:1.0:1789760783.128874:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23138304 chunk: [23138304, 23142400) 23142400, 4096 00020000:00200000:1.0:1789760783.128879:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.128902:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5649, 5649] 00000008:00010000:1.0:1789760783.128922:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.128928:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.128942:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23138304, 23142400) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.128958:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23138304=0x1611000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.129097:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.129101:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23142400 00000020:00010000:1.0:1789760783.129111:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.129114:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.129117:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.129121:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.129189:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.129201:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.129208:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.129227:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760783.129339:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23138304<>23142400 00000100:00100000:0.0:1789760783.129452:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507359616:192.168.203.141@tcp:4:dd.0 00010000:00010000:0.0:1789760783.165185:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:1.0:1789760783.165292:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972968 gen 3 00000100:00100000:1.0:1789760783.165331:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8169880 x1876699507359616/t4294972969(4294972969) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760799 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760783.165402:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:1.0:1789760783.165410:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760783.165421:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760783.165495:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:1.0:1789760783.165519:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507359616:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760783.165946:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5649, 5649] 00000080:00200000:2.0:1789760783.165963:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.166033:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.166051:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23142400 i_blocks 47111 00000020:00200000:2.0:1789760783.166056:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.166068:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.166103:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23142400, count: 4096 00000080:00200000:2.0:1789760783.166111:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23142400, bytes: 4096 00000080:00200000:2.0:1789760783.166186:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.166197:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.166199:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23142400, 23146496) 0 0 00000080:00200000:2.0:1789760783.166204:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.166221:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5650, 5650] 00020000:00200000:2.0:1789760783.166231:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23142400 chunk: [23142400, 23146496) 23146496, 4096 00020000:00200000:2.0:1789760783.166235:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.166255:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5650, 5650] 00000008:00010000:2.0:1789760783.166273:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.166279:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.166291:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23142400, 23146496) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.166306:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23142400=0x1612000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.166424:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.166427:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23146496 00000020:00010000:2.0:1789760783.166437:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.166440:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.166444:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.166447:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.166450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.166457:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.166464:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.166487:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760783.166602:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 23142400<>23146496 00000100:00100000:2.0:1789760783.200834:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972970 gen 3 00000008:00010000:2.0:1789760783.200899:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ace40 00000020:00200000:2.0:1789760783.200908:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760783.200918:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760783.200941:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ace40 00000100:00100000:2.0:1789760783.200981:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507359744:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.201146:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5650, 5650] 00000080:00200000:1.0:1789760783.201162:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.201169:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.201188:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23146496 i_blocks 47111 00000020:00200000:1.0:1789760783.201191:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.201201:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.201236:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23146496, count: 4096 00000080:00200000:1.0:1789760783.201245:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23146496, bytes: 4096 00000080:00200000:1.0:1789760783.201255:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.201265:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.201268:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23146496, 23150592) 0 0 00000080:00200000:1.0:1789760783.201272:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.201290:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5651, 5651] 00020000:00200000:1.0:1789760783.201298:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23146496 chunk: [23146496, 23150592) 23150592, 4096 00020000:00200000:1.0:1789760783.201302:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.201321:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5651, 5651] 00000008:00010000:1.0:1789760783.201341:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.201347:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.201359:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23146496, 23150592) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.201374:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23146496=0x1613000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.201469:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.201472:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23150592 00000020:00010000:1.0:1789760783.201480:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.201484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.201487:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.201490:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.201493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.201501:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.201508:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.201525:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760783.203718:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23146496<>23150592 00000100:00100000:0.0:1789760783.203800:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760783.204040:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507359872:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.243182:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972971 gen 3 00000008:00010000:0.0:1789760783.243233:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760783.243242:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.243253:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.243276:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760783.243308:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507359872:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760783.243442:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5651, 5651] 00000080:00200000:2.0:1789760783.243461:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.243466:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.243482:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23150592 i_blocks 47111 00000020:00200000:2.0:1789760783.243486:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.243497:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.243530:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23150592, count: 4096 00000080:00200000:2.0:1789760783.243538:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23150592, bytes: 4096 00000080:00200000:2.0:1789760783.243548:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.243557:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.243560:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23150592, 23154688) 0 0 00000080:00200000:2.0:1789760783.243564:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.243581:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5652, 5652] 00020000:00200000:2.0:1789760783.243590:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23150592 chunk: [23150592, 23154688) 23154688, 4096 00020000:00200000:2.0:1789760783.243594:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.243615:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5652, 5652] 00000008:00010000:2.0:1789760783.243634:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.243640:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.243651:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23150592, 23154688) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.243666:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23150592=0x1614000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.243855:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.243858:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23154688 00000020:00010000:2.0:1789760783.243868:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.243872:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.243875:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.243878:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.243880:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.243888:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.243895:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.243912:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760783.244020:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 23150592<>23154688 00000100:00100000:2.0:1789760783.244135:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760783.244154:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507360000:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760783.282697:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972972 gen 3 00000008:00010000:2.0:1789760783.282757:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ace40 00000020:00200000:2.0:1789760783.282767:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760783.282779:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760783.282795:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ace40 00000100:00100000:2.0:1789760783.282824:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507360000:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760783.283014:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5652, 5652] 00000080:00200000:0.0:1789760783.283031:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760783.283037:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760783.283068:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23154688 i_blocks 47111 00000020:00200000:0.0:1789760783.283072:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760783.283082:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760783.283132:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23154688, count: 4096 00000080:00200000:0.0:1789760783.283140:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23154688, bytes: 4096 00000080:00200000:0.0:1789760783.283155:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760783.283165:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760783.283168:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23154688, 23158784) 0 0 00000080:00200000:0.0:1789760783.283173:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760783.283190:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5653, 5653] 00020000:00200000:0.0:1789760783.283206:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23154688 chunk: [23154688, 23158784) 23158784, 4096 00020000:00200000:0.0:1789760783.283211:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760783.283231:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5653, 5653] 00000008:00010000:0.0:1789760783.283258:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760783.283264:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760783.283276:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23154688, 23158784) append 0 append lockpos 0 00000080:00200000:0.0:1789760783.283290:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23154688=0x1615000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760783.283348:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760783.283351:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23158784 00000020:00010000:0.0:1789760783.283361:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760783.283365:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760783.283368:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760783.283372:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760783.283375:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760783.283420:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760783.283437:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760783.283454:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7480 00000008:00100000:0.0:1789760783.283775:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb480 - object 0x280000401:3 offset 23154688<>23158784 00000100:00100000:0.0:1789760783.283816:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760783.283834:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507360128:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.320698:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972973 gen 3 00000008:00010000:0.0:1789760783.320748:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:0.0:1789760783.320757:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.320770:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.320792:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:0.0:1789760783.320822:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507360128:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.321419:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5653, 5653] 00000080:00200000:1.0:1789760783.321647:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.321654:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.321702:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23158784 i_blocks 47111 00000020:00200000:1.0:1789760783.321706:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.321719:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.321753:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23158784, count: 4096 00000080:00200000:1.0:1789760783.321761:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23158784, bytes: 4096 00000080:00200000:1.0:1789760783.321782:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.321791:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.321794:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23158784, 23162880) 0 0 00000080:00200000:1.0:1789760783.321798:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.321816:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5654, 5654] 00020000:00200000:1.0:1789760783.321825:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23158784 chunk: [23158784, 23162880) 23162880, 4096 00020000:00200000:1.0:1789760783.321829:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.321851:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5654, 5654] 00000008:00010000:1.0:1789760783.321871:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.321877:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.321889:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23158784, 23162880) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.321904:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23158784=0x1616000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.321966:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.321968:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23162880 00000020:00010000:1.0:1789760783.321976:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.321980:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.321983:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.321986:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.321989:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.321997:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.322004:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.322023:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760783.322132:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23158784<>23162880 00000100:00100000:1.0:1789760783.322184:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760783.322207:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360256:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.359276:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972974 gen 3 00000008:00010000:0.0:1789760783.359331:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760783.359340:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.359351:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.359369:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000080:00200000:1.0:1789760783.359400:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5654, 5654] 00000080:00200000:1.0:1789760783.359419:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.359425:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000100:00100000:0.0:1789760783.359437:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360256:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.359441:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23162880 i_blocks 47111 00000020:00200000:1.0:1789760783.359446:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.359456:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.360727:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23162880, count: 4096 00000080:00200000:1.0:1789760783.360747:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23162880, bytes: 4096 00000080:00200000:1.0:1789760783.360763:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.360772:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.360774:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23162880, 23166976) 0 0 00000080:00200000:1.0:1789760783.360779:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.360850:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5655, 5655] 00020000:00200000:1.0:1789760783.360860:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23162880 chunk: [23162880, 23166976) 23166976, 4096 00020000:00200000:1.0:1789760783.360865:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.360887:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5655, 5655] 00000008:00010000:1.0:1789760783.360905:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.360911:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.360924:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23162880, 23166976) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.360939:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23162880=0x1617000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.361104:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.361108:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23166976 00000020:00010000:1.0:1789760783.361117:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.361120:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.361124:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.361127:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.361130:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.361139:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.361146:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.361340:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760783.361805:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23162880<>23166976 00000100:00100000:0.0:1789760783.362045:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360384:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.432998:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972975 gen 3 00000008:00010000:0.0:1789760783.433097:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760783.433103:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.433114:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.433136:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760783.433165:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360384:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.433222:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5655, 5655] 00000080:00200000:1.0:1789760783.433238:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.433244:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.433260:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23166976 i_blocks 47111 00000020:00200000:1.0:1789760783.433263:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.433272:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.433307:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23166976, count: 4096 00000080:00200000:1.0:1789760783.433314:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23166976, bytes: 4096 00000080:00200000:1.0:1789760783.433323:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.433332:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.433334:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23166976, 23171072) 0 0 00000080:00200000:1.0:1789760783.433339:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.433366:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5656, 5656] 00020000:00200000:1.0:1789760783.433375:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23166976 chunk: [23166976, 23171072) 23171072, 4096 00020000:00200000:1.0:1789760783.433421:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.433447:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5656, 5656] 00000008:00010000:1.0:1789760783.433466:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.433472:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.433545:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23166976, 23171072) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.433561:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23166976=0x1618000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.433616:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.433618:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23171072 00000020:00010000:1.0:1789760783.433625:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.433627:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.433630:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.433632:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.433634:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.433640:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.433645:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.433658:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760783.433775:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23166976<>23171072 00000100:00100000:0.0:1789760783.434002:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507360512:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.489085:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972976 gen 3 00000008:00010000:0.0:1789760783.489218:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760783.489227:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.489239:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.489323:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760783.489356:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507360512:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.489512:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5656, 5656] 00000080:00200000:1.0:1789760783.489530:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.489536:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.489551:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23171072 i_blocks 47111 00000020:00200000:1.0:1789760783.489554:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.489564:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.489595:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23171072, count: 4096 00000080:00200000:1.0:1789760783.489603:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23171072, bytes: 4096 00000080:00200000:1.0:1789760783.489616:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.489624:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.489626:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23171072, 23175168) 0 0 00000080:00200000:1.0:1789760783.489631:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.489646:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5657, 5657] 00020000:00200000:1.0:1789760783.489654:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23171072 chunk: [23171072, 23175168) 23175168, 4096 00020000:00200000:1.0:1789760783.489658:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.489689:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5657, 5657] 00000008:00010000:1.0:1789760783.489883:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.489890:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.490079:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23171072, 23175168) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.490094:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23171072=0x1619000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.490215:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.490218:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23175168 00000020:00010000:1.0:1789760783.490227:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.490230:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.490233:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.490236:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.490239:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.490246:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.490252:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.490269:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760783.490363:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23171072<>23175168 00000100:00100000:0.0:1789760783.490554:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360640:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760783.525578:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760783.525595:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.525608:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.525632:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760783.525666:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360640:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.525981:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5657, 5657] 00000080:00200000:1.0:1789760783.525997:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.526004:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.526018:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23175168 i_blocks 47111 00000020:00200000:1.0:1789760783.526022:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.526032:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.526079:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23175168, count: 4096 00000080:00200000:1.0:1789760783.526087:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23175168, bytes: 4096 00000080:00200000:1.0:1789760783.526097:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.526107:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.526109:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23175168, 23179264) 0 0 00000080:00200000:1.0:1789760783.526244:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.526264:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5658, 5658] 00020000:00200000:1.0:1789760783.526273:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23175168 chunk: [23175168, 23179264) 23179264, 4096 00020000:00200000:1.0:1789760783.526278:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.526298:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5658, 5658] 00000008:00010000:1.0:1789760783.526317:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.526323:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.526335:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23175168, 23179264) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.526351:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23175168=0x161a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.526749:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.526754:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23179264 00000020:00010000:1.0:1789760783.526763:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.526767:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.526770:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.526773:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.526776:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.526784:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.526791:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.526813:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760783.526924:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 23175168<>23179264 00000100:00100000:0.0:1789760783.527006:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507360768:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.557649:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972977 gen 3 00000100:00100000:0.0:1789760783.557703:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8168a80 x1876699507360768/t4294972978(4294972978) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760799 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760783.557821:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760783.557828:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.557838:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.557862:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760783.557887:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507360768:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.558033:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5658, 5658] 00000080:00200000:1.0:1789760783.558050:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.558056:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.558071:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23179264 i_blocks 47111 00000020:00200000:1.0:1789760783.558076:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.558086:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.558193:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23179264, count: 4096 00000080:00200000:1.0:1789760783.558201:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23179264, bytes: 4096 00000080:00200000:1.0:1789760783.558213:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.558222:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.558224:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23179264, 23183360) 0 0 00000080:00200000:1.0:1789760783.558229:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.558245:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5659, 5659] 00020000:00200000:1.0:1789760783.558253:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23179264 chunk: [23179264, 23183360) 23183360, 4096 00020000:00200000:1.0:1789760783.558257:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.558277:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5659, 5659] 00000008:00010000:1.0:1789760783.558296:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.558302:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.558315:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23179264, 23183360) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.558330:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23179264=0x161b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.558425:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.558428:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23183360 00000020:00010000:1.0:1789760783.558438:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.558442:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.558446:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.558450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.558452:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.558461:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.558469:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.558593:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760783.558783:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23179264<>23183360 00000100:00100000:0.0:1789760783.558970:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360896:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.593197:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972979 gen 3 00000008:00010000:0.0:1789760783.593265:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760783.593274:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.593284:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.593309:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760783.593340:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507360896:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.593538:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5659, 5659] 00000080:00200000:1.0:1789760783.593555:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.593561:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.593576:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23183360 i_blocks 47111 00000020:00200000:1.0:1789760783.593580:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.593590:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.593625:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23183360, count: 4096 00000080:00200000:1.0:1789760783.593631:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23183360, bytes: 4096 00000080:00200000:1.0:1789760783.593643:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.593652:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.593654:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23183360, 23187456) 0 0 00000080:00200000:1.0:1789760783.593659:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.593687:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5660, 5660] 00020000:00200000:1.0:1789760783.593696:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23183360 chunk: [23183360, 23187456) 23187456, 4096 00020000:00200000:1.0:1789760783.593701:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.593733:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5660, 5660] 00000008:00010000:1.0:1789760783.593751:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.593757:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.593769:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23183360, 23187456) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.593783:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23183360=0x161c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.593844:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.593847:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23187456 00000020:00010000:1.0:1789760783.593856:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.593860:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.593863:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.593867:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.593870:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.593878:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.593885:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.593915:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760783.594026:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23183360<>23187456 00000100:00100000:0.0:1789760783.594508:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760783.594530:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507361024:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760783.636480:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760783.636494:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.636505:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.636525:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760783.636552:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507361024:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.636647:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5660, 5660] 00000080:00200000:1.0:1789760783.636665:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.636682:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.636697:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23187456 i_blocks 47111 00000020:00200000:1.0:1789760783.636700:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.636765:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.636805:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23187456, count: 4096 00000080:00200000:1.0:1789760783.636813:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23187456, bytes: 4096 00000080:00200000:1.0:1789760783.636823:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.636832:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.636834:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23187456, 23191552) 0 0 00000080:00200000:1.0:1789760783.636839:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.636856:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5661, 5661] 00020000:00200000:1.0:1789760783.636864:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23187456 chunk: [23187456, 23191552) 23191552, 4096 00020000:00200000:1.0:1789760783.636868:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.636886:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5661, 5661] 00000008:00010000:1.0:1789760783.636961:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.636967:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.636978:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23187456, 23191552) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.636991:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23187456=0x161d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.637045:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.637047:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23191552 00000020:00010000:1.0:1789760783.637055:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.637058:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.637062:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.637065:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.637068:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.637076:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.637082:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.637100:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760783.637256:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23187456<>23191552 00000100:00100000:0.0:1789760783.637351:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760783.637368:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507361152:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.669577:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972981 gen 3 00000008:00010000:0.0:1789760783.669644:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760783.669653:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.669663:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.669737:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760783.669767:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507361152:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.669849:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5661, 5661] 00000080:00200000:1.0:1789760783.669866:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.669872:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.669888:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23191552 i_blocks 47111 00000020:00200000:1.0:1789760783.669892:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.669948:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.669984:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23191552, count: 4096 00000080:00200000:1.0:1789760783.669990:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23191552, bytes: 4096 00000080:00200000:1.0:1789760783.670000:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.670008:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.670011:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23191552, 23195648) 0 0 00000080:00200000:1.0:1789760783.670016:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.670032:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5662, 5662] 00020000:00200000:1.0:1789760783.670040:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23191552 chunk: [23191552, 23195648) 23195648, 4096 00020000:00200000:1.0:1789760783.670044:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.670064:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5662, 5662] 00000008:00010000:1.0:1789760783.670083:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.670090:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.670102:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23191552, 23195648) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.670178:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23191552=0x161e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.670236:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.670239:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23195648 00000020:00010000:1.0:1789760783.670246:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.670248:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.670251:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.670253:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.670255:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.670260:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.670265:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.670279:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760783.670370:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23191552<>23195648 00000100:00100000:0.0:1789760783.670470:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760783.670488:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507361280:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.704116:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972982 gen 3 00000008:00010000:0.0:1789760783.704177:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760783.704185:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.704195:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.704217:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760783.704244:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507361280:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.704399:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5662, 5662] 00000080:00200000:1.0:1789760783.704416:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.704422:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.704438:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23195648 i_blocks 47111 00000020:00200000:1.0:1789760783.704443:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.704452:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.704488:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23195648, count: 4096 00000080:00200000:1.0:1789760783.704496:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23195648, bytes: 4096 00000080:00200000:1.0:1789760783.704505:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.704514:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.704517:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23195648, 23199744) 0 0 00000080:00200000:1.0:1789760783.704521:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.704540:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5663, 5663] 00020000:00200000:1.0:1789760783.704549:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23195648 chunk: [23195648, 23199744) 23199744, 4096 00020000:00200000:1.0:1789760783.704553:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.704574:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5663, 5663] 00000008:00010000:1.0:1789760783.704591:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.704597:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.704622:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23195648, 23199744) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.704637:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23195648=0x161f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.704718:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.704721:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23199744 00000020:00010000:1.0:1789760783.704729:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.704732:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.704735:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.704738:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.704741:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.704749:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.704755:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.704772:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760783.704877:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23195648<>23199744 00000100:00100000:1.0:1789760783.704918:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760783.704937:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507361408:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.739988:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972983 gen 3 00000008:00010000:0.0:1789760783.740039:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760783.740046:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.740057:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.740122:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760783.740155:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507361408:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.740250:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5663, 5663] 00000080:00200000:1.0:1789760783.740266:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.740272:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.740286:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23199744 i_blocks 47111 00000020:00200000:1.0:1789760783.740290:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.740300:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.740334:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23199744, count: 4096 00000080:00200000:1.0:1789760783.740341:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23199744, bytes: 4096 00000080:00200000:1.0:1789760783.740350:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.740359:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.740362:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23199744, 23203840) 0 0 00000080:00200000:1.0:1789760783.740366:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.740419:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5664, 5664] 00020000:00200000:1.0:1789760783.740429:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23199744 chunk: [23199744, 23203840) 23203840, 4096 00020000:00200000:1.0:1789760783.740433:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.740455:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5664, 5664] 00000008:00010000:1.0:1789760783.740473:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.740479:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.740491:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23199744, 23203840) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.740506:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23199744=0x1620000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.740562:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.740565:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23203840 00000020:00010000:1.0:1789760783.740575:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.740579:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.740582:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.740585:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.740588:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.740596:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.740603:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.740620:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760783.740730:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23199744<>23203840 00000100:00100000:0.0:1789760783.740790:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507361536:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.771836:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972984 gen 3 00000008:00010000:0.0:1789760783.771894:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760783.771904:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.771916:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.771938:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760783.771964:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507361536:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760783.772059:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5664, 5664] 00000080:00200000:1.0:1789760783.772076:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760783.772083:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760783.772098:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23203840 i_blocks 47111 00000020:00200000:1.0:1789760783.772102:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760783.772112:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760783.772148:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23203840, count: 4096 00000080:00200000:1.0:1789760783.772155:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23203840, bytes: 4096 00000080:00200000:1.0:1789760783.772164:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760783.772173:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760783.772176:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23203840, 23207936) 0 0 00000080:00200000:1.0:1789760783.772180:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760783.772197:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5665, 5665] 00020000:00200000:1.0:1789760783.772205:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23203840 chunk: [23203840, 23207936) 23207936, 4096 00020000:00200000:1.0:1789760783.772209:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760783.772228:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5665, 5665] 00000008:00010000:1.0:1789760783.772246:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760783.772252:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760783.772263:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23203840, 23207936) append 0 append lockpos 0 00000080:00200000:1.0:1789760783.772277:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23203840=0x1621000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760783.772328:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760783.772330:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23207936 00000020:00010000:1.0:1789760783.772337:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760783.772340:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.772342:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760783.772345:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760783.772347:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760783.772352:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760783.772358:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760783.772371:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760783.772564:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23203840<>23207936 00000100:00100000:1.0:1789760783.772655:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507361664:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760783.815873:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972985 gen 3 00000008:00010000:0.0:1789760783.815926:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760783.815934:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760783.815946:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760783.815972:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760783.816003:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507361664:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760783.816293:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5665, 5665] 00000080:00200000:2.0:1789760783.816311:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.816316:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.816333:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23207936 i_blocks 47111 00000020:00200000:2.0:1789760783.816337:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.816347:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.816427:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23207936, count: 4096 00000080:00200000:2.0:1789760783.816435:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23207936, bytes: 4096 00000080:00200000:2.0:1789760783.816447:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.816455:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.816457:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23207936, 23212032) 0 0 00000080:00200000:2.0:1789760783.816461:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.816478:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5666, 5666] 00020000:00200000:2.0:1789760783.816486:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23207936 chunk: [23207936, 23212032) 23212032, 4096 00020000:00200000:2.0:1789760783.816490:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.816510:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5666, 5666] 00000008:00010000:2.0:1789760783.816527:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.816533:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.816543:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23207936, 23212032) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.816556:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23207936=0x1622000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.816611:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.816614:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23212032 00000020:00010000:2.0:1789760783.816623:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.816627:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.816630:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.816634:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.816638:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.816646:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.816654:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.816683:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760783.816810:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 23207936<>23212032 00000080:00200000:2.0:1789760783.880992:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5666, 5666] 00000080:00200000:2.0:1789760783.881025:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.881032:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.881048:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23212032 i_blocks 47111 00000020:00200000:2.0:1789760783.881053:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.881064:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.881100:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23212032, count: 4096 00000080:00200000:2.0:1789760783.881109:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23212032, bytes: 4096 00000080:00200000:2.0:1789760783.881311:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.881321:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.881324:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23212032, 23216128) 0 0 00000080:00200000:2.0:1789760783.881328:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.881345:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5667, 5667] 00020000:00200000:2.0:1789760783.881355:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23212032 chunk: [23212032, 23216128) 23216128, 4096 00020000:00200000:2.0:1789760783.881360:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.881422:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5667, 5667] 00000008:00010000:2.0:1789760783.881441:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.881447:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.881459:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23212032, 23216128) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.881474:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23212032=0x1623000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.881536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.881540:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23216128 00000020:00010000:2.0:1789760783.881548:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.881552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.881555:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.881558:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.881562:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.881570:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.881577:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.881650:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760783.881772:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 23212032<>23216128 00000080:00200000:2.0:1789760783.955421:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5667, 5667] 00000080:00200000:2.0:1789760783.955438:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760783.955445:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760783.955460:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23216128 i_blocks 47111 00000020:00200000:2.0:1789760783.955464:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760783.955475:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760783.955513:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23216128, count: 4096 00000080:00200000:2.0:1789760783.955520:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23216128, bytes: 4096 00000080:00200000:2.0:1789760783.955532:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760783.955541:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760783.955544:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23216128, 23220224) 0 0 00000080:00200000:2.0:1789760783.955548:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760783.955567:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5668, 5668] 00020000:00200000:2.0:1789760783.955575:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23216128 chunk: [23216128, 23220224) 23220224, 4096 00020000:00200000:2.0:1789760783.955580:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760783.955602:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5668, 5668] 00000008:00010000:2.0:1789760783.955624:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760783.955632:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760783.955646:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23216128, 23220224) append 0 append lockpos 0 00000080:00200000:2.0:1789760783.955662:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23216128=0x1624000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760783.955733:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760783.955736:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23220224 00000020:00010000:2.0:1789760783.955745:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760783.955749:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.955752:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760783.955755:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760783.955759:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760783.955766:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760783.955773:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760783.955790:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760783.955893:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 23216128<>23220224 00000080:00200000:2.0:1789760784.006416:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5668, 5668] 00000080:00200000:2.0:1789760784.006433:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760784.006440:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760784.006457:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23220224 i_blocks 47111 00000020:00200000:2.0:1789760784.006462:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760784.006472:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760784.006508:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23220224, count: 4096 00000080:00200000:2.0:1789760784.006516:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23220224, bytes: 4096 00000080:00200000:2.0:1789760784.006526:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760784.006535:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760784.006537:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23220224, 23224320) 0 0 00000080:00200000:2.0:1789760784.006542:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760784.006559:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5669, 5669] 00020000:00200000:2.0:1789760784.006567:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23220224 chunk: [23220224, 23224320) 23224320, 4096 00020000:00200000:2.0:1789760784.006572:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760784.006592:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5669, 5669] 00000008:00010000:2.0:1789760784.006609:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760784.006616:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760784.006627:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23220224, 23224320) append 0 append lockpos 0 00000080:00200000:2.0:1789760784.006642:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23220224=0x1625000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760784.006720:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760784.006723:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23224320 00000020:00010000:2.0:1789760784.006731:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760784.006735:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.006738:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.006741:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760784.006745:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760784.006752:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760784.006759:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760784.006775:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760784.006882:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 23220224<>23224320 00000100:00100000:2.0:1789760784.006930:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760784.006947:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507362176:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760784.058517:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972989 gen 3 00000008:00010000:2.0:1789760784.058578:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50acc00 00000020:00200000:2.0:1789760784.058588:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760784.058599:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760784.058746:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50acc00 00000100:00100000:2.0:1789760784.058782:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507362176:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760784.059190:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5669, 5669] 00000080:00200000:0.0:1789760784.059210:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760784.059219:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760784.059235:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23224320 i_blocks 47111 00000020:00200000:0.0:1789760784.059239:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760784.059251:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760784.059287:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23224320, count: 4096 00000080:00200000:0.0:1789760784.059294:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23224320, bytes: 4096 00000080:00200000:0.0:1789760784.059308:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760784.059316:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760784.059319:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23224320, 23228416) 0 0 00000080:00200000:0.0:1789760784.059324:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760784.059342:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5670, 5670] 00020000:00200000:0.0:1789760784.059436:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23224320 chunk: [23224320, 23228416) 23228416, 4096 00020000:00200000:0.0:1789760784.059447:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760784.059470:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5670, 5670] 00000008:00010000:0.0:1789760784.059492:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760784.059498:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760784.059511:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23224320, 23228416) append 0 append lockpos 0 00000080:00200000:0.0:1789760784.059525:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23224320=0x1626000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760784.059593:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760784.059597:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23228416 00000020:00010000:0.0:1789760784.059605:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760784.059609:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760784.059612:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760784.059616:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760784.059619:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760784.059630:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760784.059637:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760784.059660:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7480 00000008:00100000:0.0:1789760784.059784:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb480 - object 0x280000401:3 offset 23224320<>23228416 00000100:00100000:1.0:1789760784.059865:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507362304:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.117457:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972990 gen 3 00000008:00010000:0.0:1789760784.117509:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7480 00000020:00200000:0.0:1789760784.117519:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.117533:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.117557:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7480 00000100:00100000:0.0:1789760784.117596:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507362304:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760784.117978:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5670, 5670] 00000080:00200000:2.0:1789760784.117995:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760784.118001:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760784.118016:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23228416 i_blocks 47111 00000020:00200000:2.0:1789760784.118021:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760784.118031:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760784.118202:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23228416, count: 4096 00000080:00200000:2.0:1789760784.118211:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23228416, bytes: 4096 00000080:00200000:2.0:1789760784.118225:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760784.118234:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760784.118236:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23228416, 23232512) 0 0 00000080:00200000:2.0:1789760784.118241:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760784.118258:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5671, 5671] 00020000:00200000:2.0:1789760784.118267:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23228416 chunk: [23228416, 23232512) 23232512, 4096 00020000:00200000:2.0:1789760784.118272:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760784.118294:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5671, 5671] 00000008:00010000:2.0:1789760784.118315:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760784.118322:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760784.118334:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23228416, 23232512) append 0 append lockpos 0 00000080:00200000:2.0:1789760784.118349:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23228416=0x1627000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760784.118441:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760784.118444:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23232512 00000020:00010000:2.0:1789760784.118453:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760784.118456:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.118459:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.118463:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760784.118466:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760784.118475:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760784.118482:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760784.118500:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760784.119151:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 23228416<>23232512 00000080:00200000:1.0:1789760784.160753:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5671, 5671] 00000080:00200000:1.0:1789760784.160770:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.160776:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.160792:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23232512 i_blocks 47111 00000020:00200000:1.0:1789760784.160795:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.160806:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.160842:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23232512, count: 4096 00000080:00200000:1.0:1789760784.160849:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23232512, bytes: 4096 00000080:00200000:1.0:1789760784.160860:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.160869:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.160871:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23232512, 23236608) 0 0 00000080:00200000:1.0:1789760784.160876:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.160893:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5672, 5672] 00020000:00200000:1.0:1789760784.160903:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23232512 chunk: [23232512, 23236608) 23236608, 4096 00020000:00200000:1.0:1789760784.160907:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.160927:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5672, 5672] 00000008:00010000:1.0:1789760784.160946:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.160951:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.160964:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23232512, 23236608) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.160978:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23232512=0x1628000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.161475:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.161480:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23236608 00000020:00010000:1.0:1789760784.161491:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.161494:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.161498:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.161503:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.161506:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.161515:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.161523:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.161546:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760784.162807:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23232512<>23236608 00000100:00100000:0.0:1789760784.162899:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507362560:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.211448:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972992 gen 3 00000008:00010000:0.0:1789760784.211501:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760784.211510:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.211522:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.211545:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760784.211575:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507362560:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.211769:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5672, 5672] 00000080:00200000:1.0:1789760784.211786:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.211792:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.211808:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23236608 i_blocks 47111 00000020:00200000:1.0:1789760784.211812:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.211823:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.211856:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23236608, count: 4096 00000080:00200000:1.0:1789760784.211865:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23236608, bytes: 4096 00000080:00200000:1.0:1789760784.211875:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.211884:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.211886:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23236608, 23240704) 0 0 00000080:00200000:1.0:1789760784.211891:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.211909:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5673, 5673] 00020000:00200000:1.0:1789760784.211918:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23236608 chunk: [23236608, 23240704) 23240704, 4096 00020000:00200000:1.0:1789760784.211923:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.211946:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5673, 5673] 00000008:00010000:1.0:1789760784.211964:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.211970:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.211982:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23236608, 23240704) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.211996:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23236608=0x1629000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.212121:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.212124:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23240704 00000020:00010000:1.0:1789760784.212134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.212137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.212140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.212145:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.212147:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.212156:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.212164:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.212181:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760784.212289:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23236608<>23240704 00000100:00100000:0.0:1789760784.212430:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507362688:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.262426:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972993 gen 3 00000008:00010000:0.0:1789760784.262480:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760784.262488:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.262500:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.262523:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760784.262553:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507362688:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.262791:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5673, 5673] 00000080:00200000:1.0:1789760784.262809:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.262815:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.262832:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23240704 i_blocks 47111 00000020:00200000:1.0:1789760784.262836:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.262846:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.262880:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23240704, count: 4096 00000080:00200000:1.0:1789760784.262889:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23240704, bytes: 4096 00000080:00200000:1.0:1789760784.262899:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.262909:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.262911:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23240704, 23244800) 0 0 00000080:00200000:1.0:1789760784.262916:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.262933:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5674, 5674] 00020000:00200000:1.0:1789760784.263561:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23240704 chunk: [23240704, 23244800) 23244800, 4096 00020000:00200000:1.0:1789760784.263570:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.263594:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5674, 5674] 00000008:00010000:1.0:1789760784.263615:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.263621:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.263634:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23240704, 23244800) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.263649:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23240704=0x162a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.264183:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.264188:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23244800 00000020:00010000:1.0:1789760784.264198:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.264202:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.264205:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.264210:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.264213:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.264223:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.264231:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.264259:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760784.264420:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23240704<>23244800 00000100:00100000:0.0:1789760784.264554:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507362816:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.309661:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972994 gen 3 00000008:00010000:0.0:1789760784.309732:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760784.309741:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.309753:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.309771:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760784.309802:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507362816:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.310021:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5674, 5674] 00000080:00200000:1.0:1789760784.310039:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.310045:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.310061:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23244800 i_blocks 47111 00000020:00200000:1.0:1789760784.310065:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.310076:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.310111:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23244800, count: 4096 00000080:00200000:1.0:1789760784.310119:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23244800, bytes: 4096 00000080:00200000:1.0:1789760784.310129:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.310141:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.310143:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23244800, 23248896) 0 0 00000080:00200000:1.0:1789760784.310149:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.310165:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5675, 5675] 00020000:00200000:1.0:1789760784.310174:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23244800 chunk: [23244800, 23248896) 23248896, 4096 00020000:00200000:1.0:1789760784.310179:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.310199:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5675, 5675] 00000008:00010000:1.0:1789760784.310216:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.310222:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.310235:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23244800, 23248896) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.310249:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23244800=0x162b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.310311:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.310314:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23248896 00000020:00010000:1.0:1789760784.310322:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.310326:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.310329:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.310333:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.310336:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.310344:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.310352:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.310370:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760784.310522:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23244800<>23248896 00000100:00100000:0.0:1789760784.310838:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507362944:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.398884:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972995 gen 3 00000008:00010000:0.0:1789760784.398944:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760784.398953:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.398964:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.399114:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760784.399148:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507362944:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.400416:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5675, 5675] 00000080:00200000:1.0:1789760784.400444:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.400451:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.400466:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23248896 i_blocks 47111 00000020:00200000:1.0:1789760784.400471:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.400484:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.400519:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23248896, count: 4096 00000080:00200000:1.0:1789760784.400528:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23248896, bytes: 4096 00000080:00200000:1.0:1789760784.400542:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.400551:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.400553:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23248896, 23252992) 0 0 00000080:00200000:1.0:1789760784.400558:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.400575:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5676, 5676] 00020000:00200000:1.0:1789760784.400583:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23248896 chunk: [23248896, 23252992) 23252992, 4096 00020000:00200000:1.0:1789760784.400588:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.400610:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5676, 5676] 00000008:00010000:1.0:1789760784.400628:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.400635:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.400647:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23248896, 23252992) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.400662:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23248896=0x162c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.403212:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.403225:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23252992 00000020:00010000:1.0:1789760784.403252:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.403258:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.403263:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.403269:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.403272:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.403286:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.403298:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.403338:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760784.403479:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23248896<>23252992 00000100:00100000:0.0:1789760784.404280:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363072:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.447124:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972996 gen 3 00000008:00010000:0.0:1789760784.447179:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760784.447188:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.447198:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.447220:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760784.447250:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363072:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.447359:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5676, 5676] 00000080:00200000:1.0:1789760784.447431:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.447438:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.447456:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23252992 i_blocks 47111 00000020:00200000:1.0:1789760784.447459:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.447470:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.447508:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23252992, count: 4096 00000080:00200000:1.0:1789760784.447516:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23252992, bytes: 4096 00000080:00200000:1.0:1789760784.447526:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.447536:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.447538:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23252992, 23257088) 0 0 00000080:00200000:1.0:1789760784.447544:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.447562:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5677, 5677] 00020000:00200000:1.0:1789760784.447571:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23252992 chunk: [23252992, 23257088) 23257088, 4096 00020000:00200000:1.0:1789760784.447576:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.447596:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5677, 5677] 00000008:00010000:1.0:1789760784.447614:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.447620:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.447633:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23252992, 23257088) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.447647:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23252992=0x162d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.447722:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.447726:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23257088 00000020:00010000:1.0:1789760784.447736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.447739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.447742:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.447746:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.447750:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.447758:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.447764:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.447781:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760784.447886:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 23252992<>23257088 00000100:00100000:0.0:1789760784.448072:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363200:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.492320:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972997 gen 3 00000008:00010000:0.0:1789760784.492367:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760784.492410:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.492422:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.492442:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760784.492470:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363200:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.492568:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5677, 5677] 00000080:00200000:1.0:1789760784.492585:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.492591:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.492608:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23257088 i_blocks 47111 00000020:00200000:1.0:1789760784.492611:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.492622:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.492655:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23257088, count: 4096 00000080:00200000:1.0:1789760784.492671:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23257088, bytes: 4096 00000080:00200000:1.0:1789760784.492681:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.492689:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.492692:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23257088, 23261184) 0 0 00000080:00200000:1.0:1789760784.492696:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.492712:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5678, 5678] 00020000:00200000:1.0:1789760784.492721:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23257088 chunk: [23257088, 23261184) 23261184, 4096 00020000:00200000:1.0:1789760784.492725:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.492746:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5678, 5678] 00000008:00010000:1.0:1789760784.492764:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.492770:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.492782:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23257088, 23261184) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.492795:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23257088=0x162e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.492852:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.492855:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23261184 00000020:00010000:1.0:1789760784.492864:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.492867:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.492871:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.492875:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.492879:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.492887:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.492894:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.492911:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760784.493012:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 23257088<>23261184 00000100:00100000:0.0:1789760784.493119:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363328:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.526256:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972998 gen 3 00000008:00010000:0.0:1789760784.526305:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760784.526314:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.526325:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.526347:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760784.526421:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363328:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.526530:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5678, 5678] 00000080:00200000:1.0:1789760784.526547:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.526553:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.526569:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23261184 i_blocks 47111 00000020:00200000:1.0:1789760784.526573:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.526583:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.526617:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23261184, count: 4096 00000080:00200000:1.0:1789760784.526624:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23261184, bytes: 4096 00000080:00200000:1.0:1789760784.526633:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.526643:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.526645:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23261184, 23265280) 0 0 00000080:00200000:1.0:1789760784.526650:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.526677:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5679, 5679] 00020000:00200000:1.0:1789760784.526686:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23261184 chunk: [23261184, 23265280) 23265280, 4096 00020000:00200000:1.0:1789760784.526705:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.526726:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5679, 5679] 00000008:00010000:1.0:1789760784.526745:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.526751:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.526762:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23261184, 23265280) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.526777:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23261184=0x162f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.526834:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.526837:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23265280 00000020:00010000:1.0:1789760784.526847:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.526850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.526854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.526857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.526860:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.526869:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.526876:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.526893:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760784.527006:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23261184<>23265280 00000100:00100000:0.0:1789760784.527554:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363456:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.600125:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294972999 gen 3 00000008:00010000:0.0:1789760784.600191:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760784.600200:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.600213:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.600236:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760784.600268:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363456:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.600430:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5679, 5679] 00000080:00200000:1.0:1789760784.600447:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.600454:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.600471:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23265280 i_blocks 47111 00000020:00200000:1.0:1789760784.600475:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.600488:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.600524:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23265280, count: 4096 00000080:00200000:1.0:1789760784.600531:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23265280, bytes: 4096 00000080:00200000:1.0:1789760784.600542:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.600552:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.600554:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23265280, 23269376) 0 0 00000080:00200000:1.0:1789760784.600559:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.600576:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5680, 5680] 00020000:00200000:1.0:1789760784.600585:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23265280 chunk: [23265280, 23269376) 23269376, 4096 00020000:00200000:1.0:1789760784.600590:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.600613:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5680, 5680] 00000008:00010000:1.0:1789760784.600635:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.600641:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.600651:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23265280, 23269376) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.600676:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23265280=0x1630000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.600842:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.600846:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23269376 00000020:00010000:1.0:1789760784.600855:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.600859:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.600862:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.600866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.600869:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.600879:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.600886:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.600915:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760784.601042:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23265280<>23269376 00000100:00100000:0.0:1789760784.601270:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363584:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.652912:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973000 gen 3 00000008:00010000:0.0:1789760784.652966:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760784.652974:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.652986:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.653008:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760784.653037:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363584:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.653355:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5680, 5680] 00000080:00200000:1.0:1789760784.653372:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.653410:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.653427:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23269376 i_blocks 47111 00000020:00200000:1.0:1789760784.653431:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.653441:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.653474:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23269376, count: 4096 00000080:00200000:1.0:1789760784.653482:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23269376, bytes: 4096 00000080:00200000:1.0:1789760784.653492:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.653501:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.653503:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23269376, 23273472) 0 0 00000080:00200000:1.0:1789760784.653508:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.653525:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5681, 5681] 00020000:00200000:1.0:1789760784.653533:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23269376 chunk: [23269376, 23273472) 23273472, 4096 00020000:00200000:1.0:1789760784.653538:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.653558:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5681, 5681] 00000008:00010000:1.0:1789760784.653576:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.653582:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.653594:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23269376, 23273472) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.653609:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23269376=0x1631000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.653676:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.653679:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23273472 00000020:00010000:1.0:1789760784.653688:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.653692:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.653695:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.653699:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.653702:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.653711:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.653717:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.653738:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760784.653841:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23269376<>23273472 00000100:00100000:0.0:1789760784.654010:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363712:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.702095:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973001 gen 3 00000008:00010000:0.0:1789760784.702157:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760784.702166:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.702177:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.702197:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760784.702227:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363712:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.702417:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5681, 5681] 00000080:00200000:1.0:1789760784.702435:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.702442:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.702458:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23273472 i_blocks 47111 00000020:00200000:1.0:1789760784.702462:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.702472:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.702509:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23273472, count: 4096 00000080:00200000:1.0:1789760784.702517:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23273472, bytes: 4096 00000080:00200000:1.0:1789760784.702530:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.702540:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.702543:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23273472, 23277568) 0 0 00000080:00200000:1.0:1789760784.702548:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.702567:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5682, 5682] 00020000:00200000:1.0:1789760784.702575:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23273472 chunk: [23273472, 23277568) 23277568, 4096 00020000:00200000:1.0:1789760784.702580:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.702601:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5682, 5682] 00000008:00010000:1.0:1789760784.702620:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.702627:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.702639:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23273472, 23277568) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.702654:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23273472=0x1632000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.702734:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.702738:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23277568 00000020:00010000:1.0:1789760784.702747:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.702750:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.702753:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.702757:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.702760:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.702768:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.702774:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.702791:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760784.702913:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23273472<>23277568 00000100:00100000:0.0:1789760784.703253:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363840:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.742875:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973002 gen 3 00000008:00010000:0.0:1789760784.742993:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760784.743004:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.743015:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.743035:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760784.743065:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507363840:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.743171:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5682, 5682] 00000080:00200000:1.0:1789760784.743188:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.743194:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.743211:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23277568 i_blocks 47111 00000020:00200000:1.0:1789760784.743215:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.743226:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.743263:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23277568, count: 4096 00000080:00200000:1.0:1789760784.743270:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23277568, bytes: 4096 00000080:00200000:1.0:1789760784.743280:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.743289:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.743291:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23277568, 23281664) 0 0 00000080:00200000:1.0:1789760784.743296:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.743313:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5683, 5683] 00020000:00200000:1.0:1789760784.743328:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23277568 chunk: [23277568, 23281664) 23281664, 4096 00020000:00200000:1.0:1789760784.743333:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.743353:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5683, 5683] 00000008:00010000:1.0:1789760784.743371:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.743406:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.743419:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23277568, 23281664) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.743433:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23277568=0x1633000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.743491:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.743494:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23281664 00000020:00010000:1.0:1789760784.743503:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.743507:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.743511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.743515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.743519:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.743527:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.743536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.743556:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760784.743782:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23277568<>23281664 00000100:00100000:0.0:1789760784.744136:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363968:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.780419:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973003 gen 3 00000008:00010000:0.0:1789760784.780471:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760784.780479:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.780489:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.780553:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760784.780583:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507363968:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.780888:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5683, 5683] 00000080:00200000:1.0:1789760784.780952:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.780959:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.780976:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23281664 i_blocks 47111 00000020:00200000:1.0:1789760784.780981:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.780992:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.781027:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23281664, count: 4096 00000080:00200000:1.0:1789760784.781034:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23281664, bytes: 4096 00000080:00200000:1.0:1789760784.781044:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.781053:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.781055:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23281664, 23285760) 0 0 00000080:00200000:1.0:1789760784.781059:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.781074:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5684, 5684] 00020000:00200000:1.0:1789760784.781082:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23281664 chunk: [23281664, 23285760) 23285760, 4096 00020000:00200000:1.0:1789760784.781087:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.781175:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5684, 5684] 00000008:00010000:1.0:1789760784.781194:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.781200:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.781212:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23281664, 23285760) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.781227:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23281664=0x1634000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.781286:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.781289:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23285760 00000020:00010000:1.0:1789760784.781297:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.781300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.781304:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.781308:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.781311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.781319:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.781326:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.781350:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760784.781677:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23281664<>23285760 00000100:00100000:0.0:1789760784.781806:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364096:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.818202:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973004 gen 3 00000008:00010000:0.0:1789760784.818258:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760784.819258:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.819287:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.819313:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760784.819349:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364096:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760784.820522:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5684, 5684] 00000080:00200000:1.0:1789760784.820547:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760784.820554:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760784.820571:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23285760 i_blocks 47111 00000020:00200000:1.0:1789760784.820576:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760784.820589:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760784.820622:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23285760, count: 4096 00000080:00200000:1.0:1789760784.820630:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23285760, bytes: 4096 00000080:00200000:1.0:1789760784.820646:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760784.820654:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760784.820657:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23285760, 23289856) 0 0 00000080:00200000:1.0:1789760784.820671:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760784.820845:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5685, 5685] 00020000:00200000:1.0:1789760784.820857:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23285760 chunk: [23285760, 23289856) 23289856, 4096 00020000:00200000:1.0:1789760784.820862:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760784.820887:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5685, 5685] 00000008:00010000:1.0:1789760784.820908:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760784.820915:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760784.820930:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23285760, 23289856) append 0 append lockpos 0 00000080:00200000:1.0:1789760784.822078:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23285760=0x1635000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760784.822165:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760784.822168:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23289856 00000020:00010000:1.0:1789760784.822177:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760784.822180:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.822184:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760784.822189:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760784.822192:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760784.822203:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760784.822211:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760784.822246:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760784.822546:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23285760<>23289856 00000100:00100000:1.0:1789760784.822612:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507364224:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.892845:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973005 gen 3 00000008:00010000:0.0:1789760784.893021:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760784.893031:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.893045:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.893076:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000080:00200000:2.0:1789760784.893476:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5685, 5685] 00000080:00200000:2.0:1789760784.893499:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760784.893505:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760784.893522:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23289856 i_blocks 47111 00000020:00200000:2.0:1789760784.893527:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760784.893539:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760784.893576:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23289856, count: 4096 00000080:00200000:2.0:1789760784.893584:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23289856, bytes: 4096 00000080:00200000:2.0:1789760784.893597:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760784.893606:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760784.893609:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23289856, 23293952) 0 0 00000080:00200000:2.0:1789760784.893613:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760784.893630:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5686, 5686] 00020000:00200000:2.0:1789760784.893638:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23289856 chunk: [23289856, 23293952) 23293952, 4096 00020000:00200000:2.0:1789760784.893643:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760784.893675:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5686, 5686] 00000008:00010000:2.0:1789760784.893695:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760784.893702:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760784.893713:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23289856, 23293952) append 0 append lockpos 0 00000080:00200000:2.0:1789760784.893726:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23289856=0x1636000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760784.893776:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760784.893779:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23293952 00000020:00010000:2.0:1789760784.893786:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760784.893790:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.893792:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.893796:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760784.893799:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760784.893807:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760784.893815:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1789760784.898092:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507364224:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760784.898167:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:0.0:1789760784.898310:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb480 - object 0x280000401:3 offset 23289856<>23293952 00000100:00100000:0.0:1789760784.898360:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364352:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760784.966645:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973006 gen 3 00000008:00010000:0.0:1789760784.966716:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:0.0:1789760784.966724:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760784.966734:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760784.966755:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:0.0:1789760784.966784:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364352:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760784.966801:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5686, 5686] 00000080:00200000:2.0:1789760784.966822:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760784.966828:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760784.966845:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23293952 i_blocks 47111 00000020:00200000:2.0:1789760784.966859:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760784.966871:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760784.966904:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23293952, count: 4096 00000080:00200000:2.0:1789760784.966912:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23293952, bytes: 4096 00000080:00200000:2.0:1789760784.966921:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760784.966931:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760784.966933:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23293952, 23298048) 0 0 00000080:00200000:2.0:1789760784.966938:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760784.966956:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5687, 5687] 00020000:00200000:2.0:1789760784.966965:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23293952 chunk: [23293952, 23298048) 23298048, 4096 00020000:00200000:2.0:1789760784.966970:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760784.966992:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5687, 5687] 00000008:00010000:2.0:1789760784.967014:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760784.967021:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760784.967043:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23293952, 23298048) append 0 append lockpos 0 00000080:00200000:2.0:1789760784.967059:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23293952=0x1637000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760784.967119:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760784.967123:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23298048 00000020:00010000:2.0:1789760784.967131:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760784.967134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.967138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760784.967140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760784.967154:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760784.967163:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760784.967170:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760784.967187:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760784.967309:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 23293952<>23298048 00000100:00100000:2.0:1789760784.967359:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507364480:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760785.019999:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973007 gen 3 00000008:00010000:2.0:1789760785.020284:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000020:00200000:2.0:1789760785.020293:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760785.020304:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760785.021169:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000100:00100000:2.0:1789760785.022125:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507364480:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.022443:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5687, 5687] 00000080:00200000:1.0:1789760785.022462:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.022468:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.022486:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23298048 i_blocks 47111 00000020:00200000:1.0:1789760785.022491:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.022502:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.022533:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23298048, count: 4096 00000080:00200000:1.0:1789760785.022540:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23298048, bytes: 4096 00000080:00200000:1.0:1789760785.022554:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.022563:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.022566:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23298048, 23302144) 0 0 00000080:00200000:1.0:1789760785.022570:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.022587:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5688, 5688] 00020000:00200000:1.0:1789760785.022596:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23298048 chunk: [23298048, 23302144) 23302144, 4096 00020000:00200000:1.0:1789760785.022601:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.022621:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5688, 5688] 00000008:00010000:1.0:1789760785.022640:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.022646:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.022658:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23298048, 23302144) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.024368:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23298048=0x1638000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.024473:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.024477:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23302144 00000020:00010000:1.0:1789760785.024489:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.024494:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.024498:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.024502:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.024505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.024517:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.024527:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.024562:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760785.024693:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23298048<>23302144 00000100:00100000:1.0:1789760785.024761:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507364608:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.057132:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973008 gen 3 00000008:00010000:0.0:1789760785.057189:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760785.057198:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.057209:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.057231:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760785.057262:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507364608:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.057415:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5688, 5688] 00000080:00200000:1.0:1789760785.057432:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.057438:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.057455:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23302144 i_blocks 47111 00000020:00200000:1.0:1789760785.057459:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.057470:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.057506:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23302144, count: 4096 00000080:00200000:1.0:1789760785.057512:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23302144, bytes: 4096 00000080:00200000:1.0:1789760785.057522:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.057531:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.057533:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23302144, 23306240) 0 0 00000080:00200000:1.0:1789760785.057538:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.057554:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5689, 5689] 00020000:00200000:1.0:1789760785.057564:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23302144 chunk: [23302144, 23306240) 23306240, 4096 00020000:00200000:1.0:1789760785.057569:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.057589:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5689, 5689] 00000008:00010000:1.0:1789760785.057606:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.057612:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.057623:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23302144, 23306240) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.057637:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23302144=0x1639000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.057712:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.057716:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23306240 00000020:00010000:1.0:1789760785.057723:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.057727:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.057730:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.057733:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.057735:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.057743:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.057750:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.057768:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760785.057875:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23302144<>23306240 00000100:00100000:0.0:1789760785.058425:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364736:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.094488:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973009 gen 3 00000008:00010000:0.0:1789760785.094536:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760785.094544:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.094556:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.094621:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760785.094650:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364736:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.094799:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5689, 5689] 00000080:00200000:1.0:1789760785.094816:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.094823:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.094839:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23306240 i_blocks 47111 00000020:00200000:1.0:1789760785.094843:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.094853:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.094890:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23306240, count: 4096 00000080:00200000:1.0:1789760785.094953:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23306240, bytes: 4096 00000080:00200000:1.0:1789760785.094967:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.094978:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.094980:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23306240, 23310336) 0 0 00000080:00200000:1.0:1789760785.094984:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.095000:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5690, 5690] 00020000:00200000:1.0:1789760785.095010:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23306240 chunk: [23306240, 23310336) 23310336, 4096 00020000:00200000:1.0:1789760785.095014:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.095036:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5690, 5690] 00000008:00010000:1.0:1789760785.095055:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.095061:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.095073:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23306240, 23310336) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.095087:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23306240=0x163a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.095214:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.095218:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23310336 00000020:00010000:1.0:1789760785.095227:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.095230:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.095233:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.095237:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.095240:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.095248:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.095255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.095276:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760785.095409:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23306240<>23310336 00000100:00100000:0.0:1789760785.095477:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507364864:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760785.130476:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760785.130494:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.130506:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.130528:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760785.130562:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507364864:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.130790:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5690, 5690] 00000080:00200000:1.0:1789760785.130806:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.130813:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.130828:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23310336 i_blocks 47111 00000020:00200000:1.0:1789760785.130832:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.130841:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.130875:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23310336, count: 4096 00000080:00200000:1.0:1789760785.130882:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23310336, bytes: 4096 00000080:00200000:1.0:1789760785.130893:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.130902:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.130904:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23310336, 23314432) 0 0 00000080:00200000:1.0:1789760785.130908:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.130924:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5691, 5691] 00020000:00200000:1.0:1789760785.130932:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23310336 chunk: [23310336, 23314432) 23314432, 4096 00020000:00200000:1.0:1789760785.130937:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.130958:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5691, 5691] 00000008:00010000:1.0:1789760785.130978:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.130984:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.130995:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23310336, 23314432) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.131010:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23310336=0x163b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.131066:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.131070:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23314432 00000020:00010000:1.0:1789760785.131078:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.131082:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.131085:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.131089:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.131092:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.131111:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.131118:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.131138:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760785.131240:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23310336<>23314432 00000100:00100000:1.0:1789760785.131289:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364992:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.177415:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973011 gen 3 00000008:00010000:0.0:1789760785.177479:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760785.177487:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.177497:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.177519:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760785.177550:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507364992:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.177650:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5691, 5691] 00000080:00200000:1.0:1789760785.177674:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.177680:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.177696:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23314432 i_blocks 47111 00000020:00200000:1.0:1789760785.177700:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.177711:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.177751:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23314432, count: 4096 00000080:00200000:1.0:1789760785.177759:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23314432, bytes: 4096 00000080:00200000:1.0:1789760785.177769:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.177777:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.177779:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23314432, 23318528) 0 0 00000080:00200000:1.0:1789760785.177784:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.177801:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5692, 5692] 00020000:00200000:1.0:1789760785.177809:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23314432 chunk: [23314432, 23318528) 23318528, 4096 00020000:00200000:1.0:1789760785.177813:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.177832:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5692, 5692] 00000008:00010000:1.0:1789760785.177850:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.177856:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.177868:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23314432, 23318528) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.177882:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23314432=0x163c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.177940:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.177943:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23318528 00000020:00010000:1.0:1789760785.177951:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.177955:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.177957:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.177961:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.177964:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.177972:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.177979:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.177997:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760785.178102:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23314432<>23318528 00000100:00100000:0.0:1789760785.178202:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365120:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.205430:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973012 gen 3 00000008:00010000:0.0:1789760785.205484:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760785.205492:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.205504:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.205523:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760785.205554:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365120:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.205939:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5692, 5692] 00000080:00200000:1.0:1789760785.205958:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.205965:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.205980:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23318528 i_blocks 47111 00000020:00200000:1.0:1789760785.205984:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.205994:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.206027:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23318528, count: 4096 00000080:00200000:1.0:1789760785.206034:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23318528, bytes: 4096 00000080:00200000:1.0:1789760785.206044:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.206053:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.206055:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23318528, 23322624) 0 0 00000080:00200000:1.0:1789760785.206060:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.206074:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5693, 5693] 00020000:00200000:1.0:1789760785.206082:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23318528 chunk: [23318528, 23322624) 23322624, 4096 00020000:00200000:1.0:1789760785.206086:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.206113:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5693, 5693] 00000008:00010000:1.0:1789760785.206131:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.206136:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.206148:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23318528, 23322624) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.206161:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23318528=0x163d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.206214:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.206217:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23322624 00000020:00010000:1.0:1789760785.206225:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.206229:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.206231:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.206235:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.206238:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.206245:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.206251:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.206267:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760785.206358:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 23318528<>23322624 00000100:00100000:0.0:1789760785.206479:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507365248:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.237286:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973013 gen 3 00000008:00010000:0.0:1789760785.237340:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760785.237349:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.237360:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.237447:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760785.237476:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507365248:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.237624:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5693, 5693] 00000080:00200000:1.0:1789760785.237643:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.237649:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.237678:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23322624 i_blocks 47111 00000020:00200000:1.0:1789760785.237683:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.237693:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.237728:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23322624, count: 4096 00000080:00200000:1.0:1789760785.237735:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23322624, bytes: 4096 00000080:00200000:1.0:1789760785.237744:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.237752:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.237754:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23322624, 23326720) 0 0 00000080:00200000:1.0:1789760785.237759:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.237776:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5694, 5694] 00020000:00200000:1.0:1789760785.237785:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23322624 chunk: [23322624, 23326720) 23326720, 4096 00020000:00200000:1.0:1789760785.237791:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.237811:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5694, 5694] 00000008:00010000:1.0:1789760785.237829:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.237834:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.237846:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23322624, 23326720) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.237861:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23322624=0x163e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.237919:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.237922:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23326720 00000020:00010000:1.0:1789760785.237931:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.237935:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.237938:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.237942:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.237945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.237952:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.237958:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.237975:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760785.238076:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 23322624<>23326720 00000100:00100000:0.0:1789760785.238468:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365376:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.271057:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973014 gen 3 00000008:00010000:0.0:1789760785.271117:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760785.271126:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.271136:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.271156:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760785.271186:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365376:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.271337:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5694, 5694] 00000080:00200000:1.0:1789760785.271353:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.271359:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.271403:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23326720 i_blocks 47111 00000020:00200000:1.0:1789760785.271407:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.271417:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.271449:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23326720, count: 4096 00000080:00200000:1.0:1789760785.271456:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23326720, bytes: 4096 00000080:00200000:1.0:1789760785.271466:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.271475:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.271477:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23326720, 23330816) 0 0 00000080:00200000:1.0:1789760785.271481:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.271498:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5695, 5695] 00020000:00200000:1.0:1789760785.271506:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23326720 chunk: [23326720, 23330816) 23330816, 4096 00020000:00200000:1.0:1789760785.271511:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.271530:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5695, 5695] 00000008:00010000:1.0:1789760785.271548:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.271554:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.271566:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23326720, 23330816) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.271579:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23326720=0x163f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.271630:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.271633:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23330816 00000020:00010000:1.0:1789760785.271640:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.271644:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.271647:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.271651:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.271653:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.271672:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.271680:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.271708:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760785.271813:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23326720<>23330816 00000100:00100000:0.0:1789760785.272049:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507365504:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760785.325459:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760785.325475:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.325487:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.325508:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760785.325538:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507365504:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.325688:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5695, 5695] 00000080:00200000:1.0:1789760785.325803:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.325810:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.325826:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23330816 i_blocks 47111 00000020:00200000:1.0:1789760785.325829:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.325839:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.325873:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23330816, count: 4096 00000080:00200000:1.0:1789760785.325880:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23330816, bytes: 4096 00000080:00200000:1.0:1789760785.325889:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.325907:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.325909:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23330816, 23334912) 0 0 00000080:00200000:1.0:1789760785.325913:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.325930:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5696, 5696] 00020000:00200000:1.0:1789760785.325939:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23330816 chunk: [23330816, 23334912) 23334912, 4096 00020000:00200000:1.0:1789760785.325943:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.325962:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5696, 5696] 00000008:00010000:1.0:1789760785.325980:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.325986:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.325997:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23330816, 23334912) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.326012:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23330816=0x1640000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.326070:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.326073:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23334912 00000020:00010000:1.0:1789760785.326082:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.326085:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.326088:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.326092:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.326095:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.326215:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.326223:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.326242:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760785.326347:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23330816<>23334912 00000100:00100000:0.0:1789760785.326428:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365632:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.357877:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973016 gen 3 00000008:00010000:0.0:1789760785.357955:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760785.357964:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.357974:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.357997:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760785.358027:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365632:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.358221:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5696, 5696] 00000080:00200000:1.0:1789760785.358238:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.358244:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.358260:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23334912 i_blocks 47111 00000020:00200000:1.0:1789760785.358264:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.358273:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.358309:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23334912, count: 4096 00000080:00200000:1.0:1789760785.358317:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23334912, bytes: 4096 00000080:00200000:1.0:1789760785.358329:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.358338:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.358341:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23334912, 23339008) 0 0 00000080:00200000:1.0:1789760785.358345:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.358361:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5697, 5697] 00020000:00200000:1.0:1789760785.358369:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23334912 chunk: [23334912, 23339008) 23339008, 4096 00020000:00200000:1.0:1789760785.358374:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.358428:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5697, 5697] 00000008:00010000:1.0:1789760785.358445:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.358451:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.358463:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23334912, 23339008) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.358477:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23334912=0x1641000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.358532:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.358536:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23339008 00000020:00010000:1.0:1789760785.358544:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.358547:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.358551:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.358554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.358557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.358566:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.358572:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.358591:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760785.358823:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23334912<>23339008 00000100:00100000:0.0:1789760785.358944:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507365760:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.389688:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973017 gen 3 00000008:00010000:0.0:1789760785.389854:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760785.389863:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.389876:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.389896:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760785.389926:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507365760:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.390029:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5697, 5697] 00000080:00200000:1.0:1789760785.390045:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.390052:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.390067:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23339008 i_blocks 47111 00000020:00200000:1.0:1789760785.390071:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.390081:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.390126:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23339008, count: 4096 00000080:00200000:1.0:1789760785.390133:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23339008, bytes: 4096 00000080:00200000:1.0:1789760785.390143:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.390153:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.390155:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23339008, 23343104) 0 0 00000080:00200000:1.0:1789760785.390160:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.390177:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5698, 5698] 00020000:00200000:1.0:1789760785.390185:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23339008 chunk: [23339008, 23343104) 23343104, 4096 00020000:00200000:1.0:1789760785.390191:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.390209:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5698, 5698] 00000008:00010000:1.0:1789760785.390226:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.390232:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.390243:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23339008, 23343104) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.390257:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23339008=0x1642000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.390314:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.390317:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23343104 00000020:00010000:1.0:1789760785.390326:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.390330:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.390334:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.390338:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.390341:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.390350:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.390356:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.390374:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760785.390525:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23339008<>23343104 00000100:00100000:0.0:1789760785.390818:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365888:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760785.427593:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760785.427607:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.427617:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.427635:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760785.427672:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507365888:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.427818:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5698, 5698] 00000080:00200000:1.0:1789760785.427834:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.427840:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.427855:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23343104 i_blocks 47111 00000020:00200000:1.0:1789760785.427858:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.427868:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.427908:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23343104, count: 4096 00000080:00200000:1.0:1789760785.427916:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23343104, bytes: 4096 00000080:00200000:1.0:1789760785.427925:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.427933:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.427936:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23343104, 23347200) 0 0 00000080:00200000:1.0:1789760785.427940:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.427955:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5699, 5699] 00020000:00200000:1.0:1789760785.427963:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23343104 chunk: [23343104, 23347200) 23347200, 4096 00020000:00200000:1.0:1789760785.427968:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.427986:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5699, 5699] 00000008:00010000:1.0:1789760785.428003:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.428008:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.428018:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23343104, 23347200) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.428030:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23343104=0x1643000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.428073:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.428076:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23347200 00000020:00010000:1.0:1789760785.428084:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.428087:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.428090:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.428094:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.428097:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.428161:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.428168:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760785.428251:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:0.0:1789760785.428353:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb480 - object 0x280000401:3 offset 23343104<>23347200 00000100:00100000:0.0:1789760785.428477:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366016:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.467869:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973019 gen 3 00000008:00010000:0.0:1789760785.467982:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760785.467991:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.468001:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.468064:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760785.468094:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366016:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.468240:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5699, 5699] 00000080:00200000:1.0:1789760785.468257:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.468263:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.468280:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23347200 i_blocks 47111 00000020:00200000:1.0:1789760785.468284:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.468294:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.468328:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23347200, count: 4096 00000080:00200000:1.0:1789760785.468335:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23347200, bytes: 4096 00000080:00200000:1.0:1789760785.468346:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.468355:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.468358:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23347200, 23351296) 0 0 00000080:00200000:1.0:1789760785.468365:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.468404:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5700, 5700] 00020000:00200000:1.0:1789760785.468413:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23347200 chunk: [23347200, 23351296) 23351296, 4096 00020000:00200000:1.0:1789760785.468418:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.468440:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5700, 5700] 00000008:00010000:1.0:1789760785.468458:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.468463:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.468474:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23347200, 23351296) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.468488:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23347200=0x1644000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.468544:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.468548:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23351296 00000020:00010000:1.0:1789760785.468556:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.468560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.468563:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.468566:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.468570:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.468578:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.468585:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.468603:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760785.468832:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23347200<>23351296 00000100:00100000:0.0:1789760785.468917:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366144:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.503877:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973020 gen 3 00000008:00010000:0.0:1789760785.503940:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760785.503950:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.503963:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.503986:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760785.504020:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366144:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.504128:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5700, 5700] 00000080:00200000:1.0:1789760785.504150:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.504159:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.504175:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23351296 i_blocks 47111 00000020:00200000:1.0:1789760785.504179:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.504190:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.504229:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23351296, count: 4096 00000080:00200000:1.0:1789760785.504235:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23351296, bytes: 4096 00000080:00200000:1.0:1789760785.504244:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.504251:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.504253:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23351296, 23355392) 0 0 00000080:00200000:1.0:1789760785.504256:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.504270:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5701, 5701] 00020000:00200000:1.0:1789760785.504277:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23351296 chunk: [23351296, 23355392) 23355392, 4096 00020000:00200000:1.0:1789760785.504281:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.504298:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5701, 5701] 00000008:00010000:1.0:1789760785.504312:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.504324:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.504334:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23351296, 23355392) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.504346:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23351296=0x1645000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.504435:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.504439:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23355392 00000020:00010000:1.0:1789760785.504447:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.504451:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.504454:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.504457:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.504460:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.504468:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.504474:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.504491:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760785.504584:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23351296<>23355392 00000100:00100000:0.0:1789760785.504633:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760785.504650:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366272:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760785.532082:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760785.532097:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.532110:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.532132:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760785.532163:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366272:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.532252:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5701, 5701] 00000080:00200000:1.0:1789760785.532269:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.532275:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.532290:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23355392 i_blocks 47111 00000020:00200000:1.0:1789760785.532294:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.532303:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.532339:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23355392, count: 4096 00000080:00200000:1.0:1789760785.532346:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23355392, bytes: 4096 00000080:00200000:1.0:1789760785.532359:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.532367:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.532370:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23355392, 23359488) 0 0 00000080:00200000:1.0:1789760785.532374:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.532412:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5702, 5702] 00020000:00200000:1.0:1789760785.532420:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23355392 chunk: [23355392, 23359488) 23359488, 4096 00020000:00200000:1.0:1789760785.532425:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.532445:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5702, 5702] 00000008:00010000:1.0:1789760785.532463:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.532469:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.532479:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23355392, 23359488) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.532493:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23355392=0x1646000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.532542:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.532546:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23359488 00000020:00010000:1.0:1789760785.532554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.532557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.532560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.532564:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.532567:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.532576:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.532582:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.532597:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760785.532816:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23355392<>23359488 00000100:00100000:0.0:1789760785.532876:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366400:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.588510:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973022 gen 3 00000008:00010000:0.0:1789760785.588577:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760785.588585:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.588595:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.588671:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760785.588702:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366400:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.589154:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5702, 5702] 00000080:00200000:1.0:1789760785.589171:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.589177:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.589193:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23359488 i_blocks 47111 00000020:00200000:1.0:1789760785.589197:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.589207:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.589241:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23359488, count: 4096 00000080:00200000:1.0:1789760785.589247:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23359488, bytes: 4096 00000080:00200000:1.0:1789760785.589259:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.589267:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.589269:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23359488, 23363584) 0 0 00000080:00200000:1.0:1789760785.589274:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.589291:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5703, 5703] 00020000:00200000:1.0:1789760785.589299:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23359488 chunk: [23359488, 23363584) 23363584, 4096 00020000:00200000:1.0:1789760785.589304:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.589324:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5703, 5703] 00000008:00010000:1.0:1789760785.589343:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.589349:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.589361:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23359488, 23363584) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.589405:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23359488=0x1647000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.589464:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.589468:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23363584 00000020:00010000:1.0:1789760785.589477:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.589481:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.589484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.589488:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.589491:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.589499:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.589507:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.589529:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760785.589758:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23359488<>23363584 00000100:00100000:0.0:1789760785.589845:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366528:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.642791:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973023 gen 3 00000008:00010000:0.0:1789760785.642846:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760785.642855:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.642867:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.642888:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760785.642917:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366528:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.643030:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5703, 5703] 00000080:00200000:1.0:1789760785.643046:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.643052:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.643068:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23363584 i_blocks 47111 00000020:00200000:1.0:1789760785.643072:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.643082:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.643130:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23363584, count: 4096 00000080:00200000:1.0:1789760785.643138:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23363584, bytes: 4096 00000080:00200000:1.0:1789760785.643148:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.643157:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.643160:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23363584, 23367680) 0 0 00000080:00200000:1.0:1789760785.643165:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.643181:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5704, 5704] 00020000:00200000:1.0:1789760785.643190:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23363584 chunk: [23363584, 23367680) 23367680, 4096 00020000:00200000:1.0:1789760785.643206:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.643228:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5704, 5704] 00000008:00010000:1.0:1789760785.643247:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.643254:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.643266:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23363584, 23367680) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.643283:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23363584=0x1648000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.643342:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.643345:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23367680 00000020:00010000:1.0:1789760785.643354:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.643358:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.643361:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.643364:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.643367:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.643418:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.643426:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.643443:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760785.643544:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23363584<>23367680 00000100:00100000:0.0:1789760785.643752:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760785.643772:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366656:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.682745:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973024 gen 3 00000008:00010000:0.0:1789760785.682794:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760785.682802:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.682812:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.682873:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760785.682916:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366656:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.683247:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5704, 5704] 00000080:00200000:1.0:1789760785.683264:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.683270:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.683289:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23367680 i_blocks 47111 00000020:00200000:1.0:1789760785.683293:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.683303:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.683339:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23367680, count: 4096 00000080:00200000:1.0:1789760785.683346:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23367680, bytes: 4096 00000080:00200000:1.0:1789760785.683356:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.683365:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.683367:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23367680, 23371776) 0 0 00000080:00200000:1.0:1789760785.683372:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.683409:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5705, 5705] 00020000:00200000:1.0:1789760785.683417:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23367680 chunk: [23367680, 23371776) 23371776, 4096 00020000:00200000:1.0:1789760785.683421:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.683440:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5705, 5705] 00000008:00010000:1.0:1789760785.683458:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.683464:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.683475:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23367680, 23371776) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.683490:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23367680=0x1649000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.683549:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.683552:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23371776 00000020:00010000:1.0:1789760785.683561:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.683564:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.683566:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.683569:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.683572:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.683582:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.683588:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.683607:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760785.683778:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23367680<>23371776 00000100:00100000:0.0:1789760785.683846:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366784:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.713443:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973025 gen 3 00000008:00010000:0.0:1789760785.713493:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760785.713502:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.713514:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.713582:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000080:00200000:1.0:1789760785.713613:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5705, 5705] 00000100:00100000:0.0:1789760785.713617:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507366784:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.713630:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.713636:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.713651:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23371776 i_blocks 47111 00000020:00200000:1.0:1789760785.713665:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.713675:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.713719:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23371776, count: 4096 00000080:00200000:1.0:1789760785.713726:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23371776, bytes: 4096 00000080:00200000:1.0:1789760785.713736:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.713745:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.713747:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23371776, 23375872) 0 0 00000080:00200000:1.0:1789760785.713752:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.713768:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5706, 5706] 00020000:00200000:1.0:1789760785.713777:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23371776 chunk: [23371776, 23375872) 23375872, 4096 00020000:00200000:1.0:1789760785.713782:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.713801:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5706, 5706] 00000008:00010000:1.0:1789760785.713819:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.713826:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.713837:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23371776, 23375872) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.713851:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23371776=0x164a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.713905:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.713909:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23375872 00000020:00010000:1.0:1789760785.713917:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.713921:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.713924:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.713927:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.713931:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.713937:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.713943:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.713959:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760785.714133:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23371776<>23375872 00000100:00100000:0.0:1789760785.715308:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366912:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760785.752187:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760785.752204:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.752217:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.752300:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760785.752334:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507366912:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.752461:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5706, 5706] 00000080:00200000:1.0:1789760785.752476:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.752482:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.752499:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23375872 i_blocks 47111 00000020:00200000:1.0:1789760785.752503:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.752513:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.752550:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23375872, count: 4096 00000080:00200000:1.0:1789760785.752558:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23375872, bytes: 4096 00000080:00200000:1.0:1789760785.752569:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.752579:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.752581:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23375872, 23379968) 0 0 00000080:00200000:1.0:1789760785.752586:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.752602:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5707, 5707] 00020000:00200000:1.0:1789760785.752612:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23375872 chunk: [23375872, 23379968) 23379968, 4096 00020000:00200000:1.0:1789760785.752617:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.752640:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5707, 5707] 00000008:00010000:1.0:1789760785.752673:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.752680:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.752693:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23375872, 23379968) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.752711:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23375872=0x164b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.752774:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.752778:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23379968 00000020:00010000:1.0:1789760785.752787:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.752791:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.752794:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.752798:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.752801:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.752809:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.752818:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.752839:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760785.753010:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23375872<>23379968 00000100:00100000:0.0:1789760785.753086:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507367040:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.795452:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973027 gen 3 00000008:00010000:0.0:1789760785.795525:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760785.795534:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.795545:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.795568:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760785.795598:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507367040:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760785.795780:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5707, 5707] 00000080:00200000:1.0:1789760785.795797:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760785.795803:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760785.795821:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23379968 i_blocks 47111 00000020:00200000:1.0:1789760785.795825:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760785.795834:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760785.795871:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23379968, count: 4096 00000080:00200000:1.0:1789760785.795878:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23379968, bytes: 4096 00000080:00200000:1.0:1789760785.795889:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760785.795953:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760785.795957:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23379968, 23384064) 0 0 00000080:00200000:1.0:1789760785.795962:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760785.795982:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5708, 5708] 00020000:00200000:1.0:1789760785.795991:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23379968 chunk: [23379968, 23384064) 23384064, 4096 00020000:00200000:1.0:1789760785.795996:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760785.796018:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5708, 5708] 00000008:00010000:1.0:1789760785.796038:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760785.796045:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760785.796058:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23379968, 23384064) append 0 append lockpos 0 00000080:00200000:1.0:1789760785.796074:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23379968=0x164c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760785.797028:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760785.797036:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23384064 00000020:00010000:1.0:1789760785.797049:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760785.797053:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.797057:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760785.797060:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760785.797063:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760785.797074:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760785.797082:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760785.797111:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760785.797461:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 23379968<>23384064 00000100:00100000:0.0:1789760785.797519:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760785.797681:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507367168:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760785.844593:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973028 gen 3 00000008:00010000:0.0:1789760785.844667:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760785.844676:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760785.844687:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760785.844712:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760785.844742:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507367168:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760785.844950:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5708, 5708] 00000080:00200000:2.0:1789760785.844968:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760785.844975:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760785.844993:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23384064 i_blocks 47111 00000020:00200000:2.0:1789760785.844997:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760785.845008:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760785.845044:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23384064, count: 4096 00000080:00200000:2.0:1789760785.845052:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23384064, bytes: 4096 00000080:00200000:2.0:1789760785.845064:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760785.845075:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760785.845078:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23384064, 23388160) 0 0 00000080:00200000:2.0:1789760785.845082:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760785.845115:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5709, 5709] 00020000:00200000:2.0:1789760785.845126:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23384064 chunk: [23384064, 23388160) 23388160, 4096 00020000:00200000:2.0:1789760785.845132:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760785.845163:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5709, 5709] 00000008:00010000:2.0:1789760785.845183:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760785.845193:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760785.845206:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23384064, 23388160) append 0 append lockpos 0 00000080:00200000:2.0:1789760785.845221:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23384064=0x164d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760785.845280:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760785.845283:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23388160 00000020:00010000:2.0:1789760785.845293:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760785.845296:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760785.845300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760785.845304:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760785.845306:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760785.845314:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760785.845322:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760785.845338:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760785.845492:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 23384064<>23388160 00000100:00100000:2.0:1789760785.845538:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760785.845556:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507367296:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760785.896986:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973029 gen 3 00000008:00010000:2.0:1789760785.897039:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000020:00200000:2.0:1789760785.897048:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760785.897059:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760785.897087:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000080:00200000:0.0:1789760785.897247:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5709, 5709] 00000080:00200000:0.0:1789760785.897267:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760785.897273:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760785.897290:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23388160 i_blocks 47111 00000020:00200000:0.0:1789760785.897294:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760785.897304:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760785.897338:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23388160, count: 4096 00000080:00200000:0.0:1789760785.897346:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23388160, bytes: 4096 00000080:00200000:0.0:1789760785.897357:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760785.897366:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760785.897369:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23388160, 23392256) 0 0 00000080:00200000:0.0:1789760785.897373:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760785.897429:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5710, 5710] 00020000:00200000:0.0:1789760785.897437:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23388160 chunk: [23388160, 23392256) 23392256, 4096 00020000:00200000:0.0:1789760785.897442:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760785.897463:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5710, 5710] 00000008:00010000:0.0:1789760785.897480:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760785.897486:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760785.897498:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23388160, 23392256) append 0 append lockpos 0 00000080:00200000:0.0:1789760785.897513:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23388160=0x164e000, pages 1 (max 131072), lockless, parallel 00000100:00100000:2.0:1789760785.897630:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507367296:192.168.203.141@tcp:4:dd.0 00000020:00200000:0.0:1789760785.898156:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760785.898162:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23392256 00000020:00010000:0.0:1789760785.898171:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760785.898175:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760785.898179:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760785.898182:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760785.898186:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760785.898195:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760785.898203:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760785.898253:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7480 00000008:00100000:2.0:1789760785.898371:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 23388160<>23392256 00000080:00200000:0.0:1789760785.947222:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5710, 5710] 00000080:00200000:0.0:1789760785.947239:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760785.947246:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760785.947262:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23392256 i_blocks 47111 00000020:00200000:0.0:1789760785.947266:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760785.947277:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760785.947310:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23392256, count: 4096 00000080:00200000:0.0:1789760785.947319:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23392256, bytes: 4096 00000080:00200000:0.0:1789760785.947329:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760785.947339:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760785.947342:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23392256, 23396352) 0 0 00000080:00200000:0.0:1789760785.947346:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760785.947364:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5711, 5711] 00020000:00200000:0.0:1789760785.947373:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23392256 chunk: [23392256, 23396352) 23396352, 4096 00020000:00200000:0.0:1789760785.947406:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760785.947427:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5711, 5711] 00000008:00010000:0.0:1789760785.947446:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760785.947452:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760785.947463:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23392256, 23396352) append 0 append lockpos 0 00000080:00200000:0.0:1789760785.947477:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23392256=0x164f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760785.947535:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760785.947538:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23396352 00000020:00010000:0.0:1789760785.947547:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760785.947551:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760785.947555:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760785.947558:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760785.947561:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760785.947570:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760785.947577:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760785.947595:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000008:00100000:0.0:1789760785.947724:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb480 - object 0x280000401:3 offset 23392256<>23396352 00000100:00100000:1.0:1789760785.947883:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760785.947905:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507367552:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760786.027200:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973031 gen 3 00000008:00010000:0.0:1789760786.027255:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:0.0:1789760786.027264:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.027276:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.027304:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:0.0:1789760786.027335:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507367552:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760786.027578:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5711, 5711] 00000080:00200000:2.0:1789760786.027596:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760786.027603:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760786.027618:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23396352 i_blocks 47111 00000020:00200000:2.0:1789760786.027622:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760786.027632:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.027679:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23396352, count: 4096 00000080:00200000:2.0:1789760786.027691:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23396352, bytes: 4096 00000080:00200000:2.0:1789760786.027704:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.027712:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.027714:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23396352, 23400448) 0 0 00000080:00200000:2.0:1789760786.027718:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.027734:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5712, 5712] 00020000:00200000:2.0:1789760786.027742:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23396352 chunk: [23396352, 23400448) 23400448, 4096 00020000:00200000:2.0:1789760786.027747:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.027767:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5712, 5712] 00000008:00010000:2.0:1789760786.027786:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.027792:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.027804:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23396352, 23400448) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.027822:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23396352=0x1650000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.027883:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.027886:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23400448 00000020:00010000:2.0:1789760786.027895:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.027899:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.027902:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.027905:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.027908:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.027918:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.027926:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.027942:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760786.028047:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 23396352<>23400448 00000080:00200000:2.0:1789760786.122684:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5712, 5712] 00000080:00200000:2.0:1789760786.122708:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760786.122714:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760786.122729:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23400448 i_blocks 47111 00000020:00200000:2.0:1789760786.122733:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760786.122743:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.122777:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23400448, count: 4096 00000080:00200000:2.0:1789760786.122784:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23400448, bytes: 4096 00000080:00200000:2.0:1789760786.122794:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.122803:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.122805:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23400448, 23404544) 0 0 00000080:00200000:2.0:1789760786.122811:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.122828:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5713, 5713] 00020000:00200000:2.0:1789760786.122836:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23400448 chunk: [23400448, 23404544) 23404544, 4096 00020000:00200000:2.0:1789760786.122841:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.122861:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5713, 5713] 00000008:00010000:2.0:1789760786.122879:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.122885:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.122898:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23400448, 23404544) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.122911:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23400448=0x1651000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.122974:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.122977:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23404544 00000020:00010000:2.0:1789760786.122985:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.122988:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.122991:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.122994:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.122997:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.123004:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.123012:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.123026:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760786.123143:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 23400448<>23404544 00000080:00200000:2.0:1789760786.164814:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5713, 5713] 00000080:00200000:2.0:1789760786.164832:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760786.164838:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760786.164854:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23404544 i_blocks 47111 00000020:00200000:2.0:1789760786.164858:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760786.164868:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.164902:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23404544, count: 4096 00000080:00200000:2.0:1789760786.164911:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23404544, bytes: 4096 00000080:00200000:2.0:1789760786.164922:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.164932:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.164934:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23404544, 23408640) 0 0 00000080:00200000:2.0:1789760786.164939:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.164957:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5714, 5714] 00020000:00200000:2.0:1789760786.164966:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23404544 chunk: [23404544, 23408640) 23408640, 4096 00020000:00200000:2.0:1789760786.164971:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.164988:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5714, 5714] 00000008:00010000:2.0:1789760786.165006:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.165013:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.165024:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23404544, 23408640) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.165038:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23404544=0x1652000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.165094:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.165097:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23408640 00000020:00010000:2.0:1789760786.165117:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.165121:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.165124:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.165128:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.165131:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.165139:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.165148:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.165165:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760786.165270:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022d80 - object 0x280000401:3 offset 23404544<>23408640 00000080:00200000:0.0:1789760786.203127:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5714, 5714] 00000080:00200000:0.0:1789760786.203144:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760786.203152:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760786.203168:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23408640 i_blocks 47111 00000020:00200000:0.0:1789760786.203173:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760786.203183:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760786.203218:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23408640, count: 4096 00000080:00200000:0.0:1789760786.203226:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23408640, bytes: 4096 00000080:00200000:0.0:1789760786.203239:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760786.203248:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760786.203251:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23408640, 23412736) 0 0 00000080:00200000:0.0:1789760786.203256:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760786.203274:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5715, 5715] 00020000:00200000:0.0:1789760786.203283:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23408640 chunk: [23408640, 23412736) 23412736, 4096 00020000:00200000:0.0:1789760786.203287:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760786.203306:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5715, 5715] 00000008:00010000:0.0:1789760786.203326:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760786.203332:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760786.203344:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23408640, 23412736) append 0 append lockpos 0 00000080:00200000:0.0:1789760786.203358:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23408640=0x1653000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760786.203476:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760786.203479:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23412736 00000020:00010000:0.0:1789760786.203489:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760786.203493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760786.203496:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760786.203500:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760786.203504:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760786.203513:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760786.203522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760786.203543:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000008:00100000:0.0:1789760786.203647:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bb480 - object 0x280000401:3 offset 23408640<>23412736 00000100:00100000:1.0:1789760786.203944:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507368064:192.168.203.141@tcp:4:dd.0 00010000:00010000:0.0:1789760786.235782:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00010000:0.0:1789760786.252025:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:0.0:1789760786.252042:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.252057:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.252084:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:0.0:1789760786.252122:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507368064:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760786.252300:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5715, 5715] 00000080:00200000:2.0:1789760786.252317:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760786.252324:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760786.252339:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23412736 i_blocks 47111 00000020:00200000:2.0:1789760786.252343:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760786.252354:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.252431:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23412736, count: 4096 00000080:00200000:2.0:1789760786.252439:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23412736, bytes: 4096 00000080:00200000:2.0:1789760786.252450:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.252459:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.252462:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23412736, 23416832) 0 0 00000080:00200000:2.0:1789760786.252467:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.252484:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5716, 5716] 00020000:00200000:2.0:1789760786.252493:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23412736 chunk: [23412736, 23416832) 23416832, 4096 00020000:00200000:2.0:1789760786.252498:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.252519:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5716, 5716] 00000008:00010000:2.0:1789760786.252538:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.252544:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.252556:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23412736, 23416832) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.252570:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23412736=0x1654000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.252632:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.252636:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23416832 00000020:00010000:2.0:1789760786.252644:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.252648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.252665:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.252669:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.252672:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.252681:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.252688:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.252704:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760786.252809:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 23412736<>23416832 00000080:00200000:2.0:1789760786.345048:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5716, 5716] 00000080:00200000:2.0:1789760786.345075:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760786.345081:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760786.345218:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23416832 i_blocks 47111 00000020:00200000:2.0:1789760786.345225:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760786.345239:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.345273:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23416832, count: 4096 00000080:00200000:2.0:1789760786.345283:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23416832, bytes: 4096 00000080:00200000:2.0:1789760786.345296:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.345305:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.345308:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23416832, 23420928) 0 0 00000080:00200000:2.0:1789760786.345313:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.345331:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5717, 5717] 00020000:00200000:2.0:1789760786.345339:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23416832 chunk: [23416832, 23420928) 23420928, 4096 00020000:00200000:2.0:1789760786.345343:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.345362:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5717, 5717] 00000008:00010000:2.0:1789760786.345781:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.345789:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.345804:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23416832, 23420928) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.345818:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23416832=0x1655000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.345890:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.345892:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23420928 00000020:00010000:2.0:1789760786.345901:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.345904:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.345908:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.345911:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.345914:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.345922:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.345930:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.345958:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760786.346065:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 23416832<>23420928 00000080:00200000:2.0:1789760786.395275:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5717, 5717] 00000080:00200000:2.0:1789760786.395291:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760786.395297:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760786.395313:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23420928 i_blocks 47111 00000020:00200000:2.0:1789760786.395317:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760786.395327:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.395361:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23420928, count: 4096 00000080:00200000:2.0:1789760786.395369:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23420928, bytes: 4096 00000080:00200000:2.0:1789760786.395478:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.395489:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.395491:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23420928, 23425024) 0 0 00000080:00200000:2.0:1789760786.395497:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.395515:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5718, 5718] 00020000:00200000:2.0:1789760786.395524:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23420928 chunk: [23420928, 23425024) 23425024, 4096 00020000:00200000:2.0:1789760786.395529:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.395549:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5718, 5718] 00000008:00010000:2.0:1789760786.395568:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.395575:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.395587:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23420928, 23425024) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.395602:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23420928=0x1656000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.395671:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.395674:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23425024 00000020:00010000:2.0:1789760786.395683:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.395686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.395690:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.395694:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.395696:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.395705:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.395712:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.395729:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760786.395828:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 23420928<>23425024 00000080:00200000:2.0:1789760786.438223:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5718, 5718] 00000080:00200000:2.0:1789760786.438242:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760786.438248:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760786.438264:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23425024 i_blocks 47111 00000020:00200000:2.0:1789760786.438268:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760786.438279:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.438315:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23425024, count: 4096 00000080:00200000:2.0:1789760786.438323:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23425024, bytes: 4096 00000080:00200000:2.0:1789760786.438334:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.438343:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.438346:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23425024, 23429120) 0 0 00000080:00200000:2.0:1789760786.438350:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.438369:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5719, 5719] 00020000:00200000:2.0:1789760786.438410:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23425024 chunk: [23425024, 23429120) 23429120, 4096 00020000:00200000:2.0:1789760786.438416:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.438435:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5719, 5719] 00000008:00010000:2.0:1789760786.438455:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.438461:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.438473:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23425024, 23429120) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.438487:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23425024=0x1657000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.438552:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.438555:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23429120 00000020:00010000:2.0:1789760786.438564:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.438568:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.438572:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.438576:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.438579:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.438588:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.438596:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.438612:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760786.439086:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23425024<>23429120 00000100:00100000:2.0:1789760786.440284:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507368576:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760786.482074:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973039 gen 3 00000008:00010000:2.0:1789760786.482126:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ace40 00000020:00200000:2.0:1789760786.482135:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760786.482147:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760786.482173:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ace40 00000100:00100000:2.0:1789760786.482203:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507368576:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760786.483115:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760786.483145:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507368704:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760786.528216:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973040 gen 3 00000008:00010000:2.0:1789760786.528269:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c3c0 00000020:00200000:2.0:1789760786.528278:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760786.528290:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760786.528314:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c3c0 00000100:00100000:2.0:1789760786.528345:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507368704:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760786.532720:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760786.532759:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23433216, count: 4096 00000080:00200000:2.0:1789760786.532768:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23433216, bytes: 4096 00000080:00200000:2.0:1789760786.532784:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760786.532794:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760786.532796:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23433216, 23437312) 0 0 00000080:00200000:2.0:1789760786.532801:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760786.532821:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5721, 5721] 00020000:00200000:2.0:1789760786.532831:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23433216 chunk: [23433216, 23437312) 23437312, 4096 00020000:00200000:2.0:1789760786.532836:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760786.532858:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5721, 5721] 00000008:00010000:2.0:1789760786.532877:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760786.532883:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760786.532895:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23433216, 23437312) append 0 append lockpos 0 00000080:00200000:2.0:1789760786.532910:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23433216=0x1659000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760786.534121:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760786.534130:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23437312 00000020:00010000:2.0:1789760786.534145:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760786.534150:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.534154:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760786.534158:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760786.534161:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760786.534174:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760786.534184:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760786.534257:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760786.534367:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23433216<>23437312 00000100:00100000:2.0:1789760786.534442:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760786.534466:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507368832:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760786.564305:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973041 gen 3 00000008:00010000:2.0:1789760786.564359:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ace40 00000020:00200000:2.0:1789760786.564368:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760786.564412:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760786.564442:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ace40 00000100:00100000:2.0:1789760786.564471:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507368832:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760786.564593:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5721, 5721] 00000080:00200000:0.0:1789760786.564612:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760786.564619:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760786.564636:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23437312 i_blocks 47111 00000020:00200000:0.0:1789760786.564640:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760786.564660:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760786.564813:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23437312, count: 4096 00000080:00200000:0.0:1789760786.564821:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23437312, bytes: 4096 00000080:00200000:0.0:1789760786.564834:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760786.564844:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760786.564846:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23437312, 23441408) 0 0 00000080:00200000:0.0:1789760786.564851:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760786.564869:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5722, 5722] 00020000:00200000:0.0:1789760786.564877:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23437312 chunk: [23437312, 23441408) 23441408, 4096 00020000:00200000:0.0:1789760786.564882:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760786.564903:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5722, 5722] 00000008:00010000:0.0:1789760786.564923:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760786.564929:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760786.564941:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23437312, 23441408) append 0 append lockpos 0 00000080:00200000:0.0:1789760786.564967:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23437312=0x165a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760786.565230:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760786.565234:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23441408 00000020:00010000:0.0:1789760786.565243:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760786.565247:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760786.565249:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760786.565253:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760786.565256:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760786.565265:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760786.565273:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760786.565292:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000008:00100000:0.0:1789760786.565431:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23437312<>23441408 00000100:00100000:0.0:1789760786.565478:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760786.565499:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507368960:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760786.603526:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:0.0:1789760786.603543:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.603555:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.603578:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000100:00100000:0.0:1789760786.603608:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507368960:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760786.603819:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5722, 5722] 00000080:00200000:1.0:1789760786.603836:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760786.603842:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760786.603857:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23441408 i_blocks 47111 00000020:00200000:1.0:1789760786.603861:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760786.603870:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760786.603956:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23441408, count: 4096 00000080:00200000:1.0:1789760786.603964:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23441408, bytes: 4096 00000080:00200000:1.0:1789760786.603974:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760786.603983:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760786.603985:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23441408, 23445504) 0 0 00000080:00200000:1.0:1789760786.603989:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760786.604006:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5723, 5723] 00020000:00200000:1.0:1789760786.604014:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23441408 chunk: [23441408, 23445504) 23445504, 4096 00020000:00200000:1.0:1789760786.604019:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760786.604038:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5723, 5723] 00000008:00010000:1.0:1789760786.604055:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760786.604061:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760786.604072:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23441408, 23445504) append 0 append lockpos 0 00000080:00200000:1.0:1789760786.604087:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23441408=0x165b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760786.604233:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760786.604236:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23445504 00000020:00010000:1.0:1789760786.604245:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760786.604249:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.604252:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.604255:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760786.604258:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760786.604265:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760786.604272:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760786.604290:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760786.604442:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 23441408<>23445504 00000100:00100000:1.0:1789760786.604473:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760786.604489:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369088:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760786.639163:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973043 gen 3 00000008:00010000:0.0:1789760786.639235:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760786.639244:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.639256:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.639277:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760786.639308:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369088:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760786.639371:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5723, 5723] 00000080:00200000:1.0:1789760786.639434:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760786.639440:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760786.639456:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23445504 i_blocks 47111 00000020:00200000:1.0:1789760786.639461:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760786.639471:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760786.639508:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23445504, count: 4096 00000080:00200000:1.0:1789760786.639516:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23445504, bytes: 4096 00000080:00200000:1.0:1789760786.639526:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760786.639537:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760786.639539:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23445504, 23449600) 0 0 00000080:00200000:1.0:1789760786.639544:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760786.639559:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5724, 5724] 00020000:00200000:1.0:1789760786.639569:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23445504 chunk: [23445504, 23449600) 23449600, 4096 00020000:00200000:1.0:1789760786.639574:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760786.639595:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5724, 5724] 00000008:00010000:1.0:1789760786.639624:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760786.639631:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760786.639643:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23445504, 23449600) append 0 append lockpos 0 00000080:00200000:1.0:1789760786.639668:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23445504=0x165c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760786.640060:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760786.640064:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23449600 00000020:00010000:1.0:1789760786.640075:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760786.640078:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.640081:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.640085:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760786.640088:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760786.640107:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760786.640115:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760786.640137:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760786.640356:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23445504<>23449600 00000100:00100000:0.0:1789760786.640531:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760786.640555:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369216:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760786.687801:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973044 gen 3 00000008:00010000:0.0:1789760786.687857:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760786.687866:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.687876:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.687997:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760786.688029:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369216:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760786.688239:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5724, 5724] 00000080:00200000:1.0:1789760786.688255:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760786.688262:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760786.688279:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23449600 i_blocks 47111 00000020:00200000:1.0:1789760786.688283:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760786.688294:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760786.688329:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23449600, count: 4096 00000080:00200000:1.0:1789760786.688337:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23449600, bytes: 4096 00000080:00200000:1.0:1789760786.688346:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760786.688355:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760786.688358:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23449600, 23453696) 0 0 00000080:00200000:1.0:1789760786.688362:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760786.688496:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5725, 5725] 00020000:00200000:1.0:1789760786.688506:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23449600 chunk: [23449600, 23453696) 23453696, 4096 00020000:00200000:1.0:1789760786.688512:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760786.688533:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5725, 5725] 00000008:00010000:1.0:1789760786.688554:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760786.688559:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760786.688571:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23449600, 23453696) append 0 append lockpos 0 00000080:00200000:1.0:1789760786.688585:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23449600=0x165d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760786.688639:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760786.688642:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23453696 00000020:00010000:1.0:1789760786.688660:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760786.688664:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.688667:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.688671:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760786.688673:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760786.688694:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760786.688701:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760786.688720:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760786.688832:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23449600<>23453696 00000100:00100000:0.0:1789760786.689039:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760786.689058:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369344:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760786.716670:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973045 gen 3 00000008:00010000:0.0:1789760786.716809:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760786.716818:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.716827:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.716845:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760786.716872:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369344:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760786.716976:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5725, 5725] 00000080:00200000:1.0:1789760786.716991:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760786.716997:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760786.717012:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23453696 i_blocks 47111 00000020:00200000:1.0:1789760786.717015:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760786.717025:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760786.717058:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23453696, count: 4096 00000080:00200000:1.0:1789760786.717064:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23453696, bytes: 4096 00000080:00200000:1.0:1789760786.717074:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760786.717082:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760786.717084:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23453696, 23457792) 0 0 00000080:00200000:1.0:1789760786.717088:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760786.717115:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5726, 5726] 00020000:00200000:1.0:1789760786.717124:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23453696 chunk: [23453696, 23457792) 23457792, 4096 00020000:00200000:1.0:1789760786.717129:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760786.717146:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5726, 5726] 00000008:00010000:1.0:1789760786.717163:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760786.717168:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760786.717179:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23453696, 23457792) append 0 append lockpos 0 00000080:00200000:1.0:1789760786.717193:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23453696=0x165e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760786.717245:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760786.717247:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23457792 00000020:00010000:1.0:1789760786.717255:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760786.717258:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.717261:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.717264:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760786.717267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760786.717274:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760786.717285:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760786.717300:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760786.717434:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23453696<>23457792 00000100:00100000:0.0:1789760786.717553:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760786.717570:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369472:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760786.752557:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973046 gen 3 00000008:00010000:0.0:1789760786.752610:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760786.752618:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.752630:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.752666:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760786.752701:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369472:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760786.752827:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5726, 5726] 00000080:00200000:1.0:1789760786.752844:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760786.752851:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760786.752868:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23457792 i_blocks 47111 00000020:00200000:1.0:1789760786.752871:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760786.752882:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760786.752915:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23457792, count: 4096 00000080:00200000:1.0:1789760786.752922:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23457792, bytes: 4096 00000080:00200000:1.0:1789760786.752937:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760786.752946:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760786.752948:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23457792, 23461888) 0 0 00000080:00200000:1.0:1789760786.752952:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760786.752970:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5727, 5727] 00020000:00200000:1.0:1789760786.752979:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23457792 chunk: [23457792, 23461888) 23461888, 4096 00020000:00200000:1.0:1789760786.752984:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760786.753004:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5727, 5727] 00000008:00010000:1.0:1789760786.753021:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760786.753027:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760786.753039:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23457792, 23461888) append 0 append lockpos 0 00000080:00200000:1.0:1789760786.753052:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23457792=0x165f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760786.753369:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760786.753372:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23461888 00000020:00010000:1.0:1789760786.753608:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760786.753612:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.753615:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.753619:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760786.753622:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760786.753631:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760786.753638:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760786.753671:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760786.753785:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23457792<>23461888 00000100:00100000:0.0:1789760786.753893:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760786.753916:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369600:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760786.828749:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973047 gen 3 00000008:00010000:0.0:1789760786.828808:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760786.828816:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.828828:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.828841:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760786.828868:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369600:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760786.828966:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5727, 5727] 00000080:00200000:1.0:1789760786.828984:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760786.828991:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760786.829009:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23461888 i_blocks 47111 00000020:00200000:1.0:1789760786.829013:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760786.829024:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760786.829066:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23461888, count: 4096 00000080:00200000:1.0:1789760786.829073:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23461888, bytes: 4096 00000080:00200000:1.0:1789760786.829084:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760786.829094:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760786.829096:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23461888, 23465984) 0 0 00000080:00200000:1.0:1789760786.829100:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760786.829119:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5728, 5728] 00020000:00200000:1.0:1789760786.829194:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23461888 chunk: [23461888, 23465984) 23465984, 4096 00020000:00200000:1.0:1789760786.829202:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760786.829222:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5728, 5728] 00000008:00010000:1.0:1789760786.829742:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760786.829750:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760786.829765:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23461888, 23465984) append 0 append lockpos 0 00000080:00200000:1.0:1789760786.829779:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23461888=0x1660000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760786.829848:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760786.829851:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23465984 00000020:00010000:1.0:1789760786.829859:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760786.829863:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.829865:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.829869:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760786.829872:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760786.829881:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760786.829889:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760786.829910:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760786.830021:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23461888<>23465984 00000100:00100000:0.0:1789760786.830822:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760786.830856:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369728:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760786.935422:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973048 gen 3 00000008:00010000:0.0:1789760786.935474:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760786.935482:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760786.935493:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760786.935513:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760786.935542:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369728:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760786.935710:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5728, 5728] 00000080:00200000:1.0:1789760786.935790:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760786.935797:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760786.935819:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23465984 i_blocks 47111 00000020:00200000:1.0:1789760786.935822:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760786.935832:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760786.935863:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23465984, count: 4096 00000080:00200000:1.0:1789760786.935870:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23465984, bytes: 4096 00000080:00200000:1.0:1789760786.935879:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760786.935887:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760786.935890:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23465984, 23470080) 0 0 00000080:00200000:1.0:1789760786.935894:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760786.935909:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5729, 5729] 00020000:00200000:1.0:1789760786.935917:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23465984 chunk: [23465984, 23470080) 23470080, 4096 00020000:00200000:1.0:1789760786.935923:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760786.935941:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5729, 5729] 00000008:00010000:1.0:1789760786.935959:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760786.935965:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760786.935977:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23465984, 23470080) append 0 append lockpos 0 00000080:00200000:1.0:1789760786.935991:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23465984=0x1661000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760786.936050:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760786.936053:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23470080 00000020:00010000:1.0:1789760786.936061:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760786.936064:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.936068:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760786.936071:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760786.936074:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760786.936082:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760786.936591:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760786.936614:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760786.937087:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23465984<>23470080 00000100:00100000:0.0:1789760786.939008:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760786.939045:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369856:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.007112:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973049 gen 3 00000008:00010000:0.0:1789760787.007159:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760787.007168:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.007180:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.007204:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760787.007235:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507369856:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.007635:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5729, 5729] 00000080:00200000:1.0:1789760787.007663:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.007670:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.007751:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23470080 i_blocks 47111 00000020:00200000:1.0:1789760787.007756:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.007766:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.007801:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23470080, count: 4096 00000080:00200000:1.0:1789760787.007808:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23470080, bytes: 4096 00000080:00200000:1.0:1789760787.007823:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.007832:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.007835:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23470080, 23474176) 0 0 00000080:00200000:1.0:1789760787.007839:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.007856:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5730, 5730] 00020000:00200000:1.0:1789760787.007864:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23470080 chunk: [23470080, 23474176) 23474176, 4096 00020000:00200000:1.0:1789760787.007870:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.007888:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5730, 5730] 00000008:00010000:1.0:1789760787.007966:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.007973:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.007986:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23470080, 23474176) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.008064:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23470080=0x1662000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.008185:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.008189:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23474176 00000020:00010000:1.0:1789760787.008197:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.008200:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.008203:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.008207:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.008210:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.008218:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.008224:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.008244:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760787.008360:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23470080<>23474176 00000100:00100000:0.0:1789760787.008575:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760787.008597:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369984:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.039557:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973050 gen 3 00000008:00010000:0.0:1789760787.039607:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760787.039615:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.039626:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.039663:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000080:00200000:2.0:1789760787.039731:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5730, 5730] 00000080:00200000:2.0:1789760787.039748:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760787.039754:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760787.039770:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23474176 i_blocks 47111 00000020:00200000:2.0:1789760787.039773:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760787.039784:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760787.039818:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23474176, count: 4096 00000080:00200000:2.0:1789760787.039827:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23474176, bytes: 4096 00000080:00200000:2.0:1789760787.039838:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760787.039847:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760787.039849:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23474176, 23478272) 0 0 00000080:00200000:2.0:1789760787.039854:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760787.039871:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5731, 5731] 00020000:00200000:2.0:1789760787.039879:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23474176 chunk: [23474176, 23478272) 23478272, 4096 00020000:00200000:2.0:1789760787.039884:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760787.039916:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5731, 5731] 00000008:00010000:2.0:1789760787.039934:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760787.039940:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760787.039951:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23474176, 23478272) append 0 append lockpos 0 00000080:00200000:2.0:1789760787.039964:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23474176=0x1663000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760787.040016:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760787.040019:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23478272 00000020:00010000:2.0:1789760787.040027:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760787.040031:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.040034:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.040038:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760787.040041:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760787.040048:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760787.040055:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:0.0:1789760787.040533:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507369984:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760787.040578:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:0.0:1789760787.040751:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23474176<>23478272 00000100:00100000:0.0:1789760787.040804:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507370112:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.088349:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973051 gen 3 00000008:00010000:0.0:1789760787.088440:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ace40 00000020:00200000:0.0:1789760787.088449:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.088462:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.088484:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ace40 00000100:00100000:0.0:1789760787.088513:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507370112:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760787.088629:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5731, 5731] 00000080:00200000:2.0:1789760787.088655:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760787.088662:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760787.088686:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23478272 i_blocks 47111 00000020:00200000:2.0:1789760787.088690:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760787.088701:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760787.088734:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23478272, count: 4096 00000080:00200000:2.0:1789760787.088742:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23478272, bytes: 4096 00000080:00200000:2.0:1789760787.088752:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760787.088760:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760787.088762:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23478272, 23482368) 0 0 00000080:00200000:2.0:1789760787.088767:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760787.088782:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5732, 5732] 00020000:00200000:2.0:1789760787.088791:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23478272 chunk: [23478272, 23482368) 23482368, 4096 00020000:00200000:2.0:1789760787.088796:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760787.088816:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5732, 5732] 00000008:00010000:2.0:1789760787.088835:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760787.088841:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760787.088852:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23478272, 23482368) append 0 append lockpos 0 00000080:00200000:2.0:1789760787.088867:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23478272=0x1664000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760787.088921:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760787.088924:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23482368 00000020:00010000:2.0:1789760787.088933:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760787.088936:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.088939:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.088942:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760787.088945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760787.088952:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760787.088959:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760787.088976:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760787.089082:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23478272<>23482368 00000080:00200000:2.0:1789760787.123592:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5732, 5732] 00000080:00200000:2.0:1789760787.123609:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760787.123615:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760787.123631:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23482368 i_blocks 47111 00000020:00200000:2.0:1789760787.123635:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760787.123655:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760787.123703:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23482368, count: 4096 00000080:00200000:2.0:1789760787.123711:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23482368, bytes: 4096 00000080:00200000:2.0:1789760787.123722:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760787.123731:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760787.123733:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23482368, 23486464) 0 0 00000080:00200000:2.0:1789760787.123738:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760787.123754:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5733, 5733] 00020000:00200000:2.0:1789760787.123763:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23482368 chunk: [23482368, 23486464) 23486464, 4096 00020000:00200000:2.0:1789760787.123768:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760787.123789:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5733, 5733] 00000008:00010000:2.0:1789760787.123808:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760787.123814:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760787.123826:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23482368, 23486464) append 0 append lockpos 0 00000080:00200000:2.0:1789760787.123840:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23482368=0x1665000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760787.123911:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760787.123914:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23486464 00000020:00010000:2.0:1789760787.123923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760787.123926:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.123929:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.123932:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760787.123935:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760787.123943:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760787.123949:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760787.123965:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760787.124066:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23482368<>23486464 00000100:00100000:2.0:1789760787.158117:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973053 gen 3 00000008:00010000:2.0:1789760787.158166:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac900 00000020:00200000:2.0:1789760787.158175:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760787.158187:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760787.158214:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac900 00000100:00100000:2.0:1789760787.158245:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507370368:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.158456:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5733, 5733] 00000080:00200000:1.0:1789760787.158474:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.158481:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.158497:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23486464 i_blocks 47111 00000020:00200000:1.0:1789760787.158501:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.158511:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.158547:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23486464, count: 4096 00000080:00200000:1.0:1789760787.158554:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23486464, bytes: 4096 00000080:00200000:1.0:1789760787.158568:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.158578:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.158580:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23486464, 23490560) 0 0 00000080:00200000:1.0:1789760787.158585:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.158601:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5734, 5734] 00020000:00200000:1.0:1789760787.158610:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23486464 chunk: [23486464, 23490560) 23490560, 4096 00020000:00200000:1.0:1789760787.158616:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.158637:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5734, 5734] 00000008:00010000:1.0:1789760787.158673:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.158680:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.158692:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23486464, 23490560) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.158706:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23486464=0x1666000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.158766:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.158770:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23490560 00000020:00010000:1.0:1789760787.158793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.158796:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.158799:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.158803:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.158807:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.158816:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.158823:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.158843:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760787.158959:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23486464<>23490560 00000100:00100000:0.0:1789760787.159722:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507370496:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.184128:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973054 gen 3 00000008:00010000:0.0:1789760787.184191:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760787.184200:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.184212:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.184354:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760787.184419:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507370496:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.184629:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5734, 5734] 00000080:00200000:1.0:1789760787.184656:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.184662:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.184844:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23490560 i_blocks 47111 00000020:00200000:1.0:1789760787.184849:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.184863:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.184903:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23490560, count: 4096 00000080:00200000:1.0:1789760787.184911:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23490560, bytes: 4096 00000080:00200000:1.0:1789760787.184922:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.184931:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.184933:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23490560, 23494656) 0 0 00000080:00200000:1.0:1789760787.184938:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.184961:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5735, 5735] 00020000:00200000:1.0:1789760787.184971:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23490560 chunk: [23490560, 23494656) 23494656, 4096 00020000:00200000:1.0:1789760787.184977:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.184999:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5735, 5735] 00000008:00010000:1.0:1789760787.185017:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.185025:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.185039:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23490560, 23494656) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.185055:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23490560=0x1667000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.185125:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.185128:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23494656 00000020:00010000:1.0:1789760787.185137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.185141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.185144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.185148:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.185151:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.185160:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.185168:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.185190:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760787.185313:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23490560<>23494656 00000100:00100000:0.0:1789760787.185419:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507370624:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760787.219782:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760787.219801:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.219814:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.219836:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760787.219868:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507370624:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.220499:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5735, 5735] 00000080:00200000:1.0:1789760787.220517:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.220522:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.220537:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23494656 i_blocks 47111 00000020:00200000:1.0:1789760787.220540:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.220550:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.220585:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23494656, count: 4096 00000080:00200000:1.0:1789760787.220591:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23494656, bytes: 4096 00000080:00200000:1.0:1789760787.220602:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.220610:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.220613:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23494656, 23498752) 0 0 00000080:00200000:1.0:1789760787.220617:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.220632:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5736, 5736] 00020000:00200000:1.0:1789760787.220640:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23494656 chunk: [23494656, 23498752) 23498752, 4096 00020000:00200000:1.0:1789760787.220658:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.220684:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5736, 5736] 00000008:00010000:1.0:1789760787.220702:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.220708:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.220720:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23494656, 23498752) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.220736:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23494656=0x1668000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.220796:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.220799:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23498752 00000020:00010000:1.0:1789760787.220808:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.220812:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.220815:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.220818:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.220821:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.220829:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.220837:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.220857:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760787.220978:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23494656<>23498752 00000100:00100000:0.0:1789760787.221246:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507370752:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.263926:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973055 gen 3 00000100:00100000:0.0:1789760787.263974:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8169c00 x1876699507370752/t4294973056(4294973056) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760803 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760787.264026:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760787.264033:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.264044:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.264167:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760787.264195:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507370752:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.265459:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5736, 5736] 00000080:00200000:1.0:1789760787.265484:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.265491:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.265510:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23498752 i_blocks 47111 00000020:00200000:1.0:1789760787.265515:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.265527:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.265564:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23498752, count: 4096 00000080:00200000:1.0:1789760787.265572:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23498752, bytes: 4096 00000080:00200000:1.0:1789760787.265584:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.265593:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.265596:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23498752, 23502848) 0 0 00000080:00200000:1.0:1789760787.265601:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.265618:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5737, 5737] 00020000:00200000:1.0:1789760787.265627:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23498752 chunk: [23498752, 23502848) 23502848, 4096 00020000:00200000:1.0:1789760787.265633:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.265665:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5737, 5737] 00000008:00010000:1.0:1789760787.265854:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.265860:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.265874:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23498752, 23502848) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.265890:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23498752=0x1669000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.266018:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.266021:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23502848 00000020:00010000:1.0:1789760787.266030:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.266033:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.266036:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.266040:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.266042:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.266052:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.266059:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.266151:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760787.266269:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23498752<>23502848 00000100:00100000:0.0:1789760787.266360:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507370880:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.314019:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973057 gen 3 00000008:00010000:0.0:1789760787.314088:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760787.314107:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.314119:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.314141:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760787.314175:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507370880:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760787.314336:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5737, 5737] 00000080:00200000:2.0:1789760787.314353:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760787.314360:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760787.314408:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23502848 i_blocks 47111 00000020:00200000:2.0:1789760787.314412:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760787.314424:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760787.314460:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23502848, count: 4096 00000080:00200000:2.0:1789760787.314468:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23502848, bytes: 4096 00000080:00200000:2.0:1789760787.314478:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760787.314487:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760787.314490:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23502848, 23506944) 0 0 00000080:00200000:2.0:1789760787.314494:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760787.314512:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5738, 5738] 00020000:00200000:2.0:1789760787.314520:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23502848 chunk: [23502848, 23506944) 23506944, 4096 00020000:00200000:2.0:1789760787.314525:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760787.314546:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5738, 5738] 00000008:00010000:2.0:1789760787.314565:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760787.314570:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760787.314582:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23502848, 23506944) append 0 append lockpos 0 00000080:00200000:2.0:1789760787.314597:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23502848=0x166a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760787.314666:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760787.314669:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23506944 00000020:00010000:2.0:1789760787.314687:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760787.314691:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.314694:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760787.314697:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760787.314701:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760787.314709:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760787.314716:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760787.314732:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760787.314839:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23502848<>23506944 00000100:00100000:2.0:1789760787.314890:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760787.314909:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507371008:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760787.358873:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973058 gen 3 00000008:00010000:2.0:1789760787.358927:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac900 00000020:00200000:2.0:1789760787.358937:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760787.358949:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760787.358980:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac900 00000100:00100000:2.0:1789760787.359020:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507371008:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760787.359138:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5738, 5738] 00000080:00200000:0.0:1789760787.359156:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760787.359163:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760787.359179:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23506944 i_blocks 47111 00000020:00200000:0.0:1789760787.359184:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760787.359194:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760787.359229:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23506944, count: 4096 00000080:00200000:0.0:1789760787.359237:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23506944, bytes: 4096 00000080:00200000:0.0:1789760787.359247:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760787.359256:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760787.359258:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23506944, 23511040) 0 0 00000080:00200000:0.0:1789760787.359263:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760787.359282:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5739, 5739] 00020000:00200000:0.0:1789760787.359291:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23506944 chunk: [23506944, 23511040) 23511040, 4096 00020000:00200000:0.0:1789760787.359296:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760787.359317:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5739, 5739] 00000008:00010000:0.0:1789760787.359337:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760787.359343:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760787.359355:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23506944, 23511040) append 0 append lockpos 0 00000080:00200000:0.0:1789760787.359369:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23506944=0x166b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760787.359459:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760787.359462:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23511040 00000020:00010000:0.0:1789760787.359470:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760787.359474:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760787.359477:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760787.359481:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760787.359484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760787.359492:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760787.359499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760787.359516:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000008:00100000:0.0:1789760787.359624:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23506944<>23511040 00000100:00100000:0.0:1789760787.359675:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760787.359766:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507371136:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.402058:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973059 gen 3 00000008:00010000:0.0:1789760787.402111:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000020:00200000:0.0:1789760787.402121:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.402133:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.406213:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000080:00200000:1.0:1789760787.406236:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5739, 5739] 00000080:00200000:1.0:1789760787.406258:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.406265:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000100:00100000:0.0:1789760787.406271:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507371136:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.406288:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23511040 i_blocks 47111 00000020:00200000:1.0:1789760787.406295:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.406308:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.406346:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23511040, count: 4096 00000080:00200000:1.0:1789760787.406354:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23511040, bytes: 4096 00000080:00200000:1.0:1789760787.406370:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.406398:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.406401:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23511040, 23515136) 0 0 00000080:00200000:1.0:1789760787.406405:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.406423:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5740, 5740] 00020000:00200000:1.0:1789760787.406432:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23511040 chunk: [23511040, 23515136) 23515136, 4096 00020000:00200000:1.0:1789760787.406437:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.406458:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5740, 5740] 00000008:00010000:1.0:1789760787.406478:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.406485:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.406496:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23511040, 23515136) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.406511:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23511040=0x166c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.406579:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.406583:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23515136 00000020:00010000:1.0:1789760787.406591:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.406595:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.406598:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.406601:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.406605:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.406614:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.406622:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.406653:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760787.406853:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23511040<>23515136 00000100:00100000:1.0:1789760787.406965:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760787.406986:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371264:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.447607:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973060 gen 3 00000008:00010000:0.0:1789760787.447680:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760787.447689:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.447701:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.447723:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760787.447755:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371264:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.447778:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5740, 5740] 00000080:00200000:1.0:1789760787.447797:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.447804:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.447821:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23515136 i_blocks 47111 00000020:00200000:1.0:1789760787.447825:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.447835:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.447883:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23515136, count: 4096 00000080:00200000:1.0:1789760787.447891:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23515136, bytes: 4096 00000080:00200000:1.0:1789760787.447900:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.447909:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.447912:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23515136, 23519232) 0 0 00000080:00200000:1.0:1789760787.447917:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.448030:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5741, 5741] 00020000:00200000:1.0:1789760787.448040:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23515136 chunk: [23515136, 23519232) 23519232, 4096 00020000:00200000:1.0:1789760787.448045:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.448067:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5741, 5741] 00000008:00010000:1.0:1789760787.448102:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.448108:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.448120:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23515136, 23519232) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.448133:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23515136=0x166d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.448188:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.448192:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23519232 00000020:00010000:1.0:1789760787.448200:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.448203:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.448207:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.448210:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.448212:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.448220:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.448227:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.448244:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760787.448352:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 23515136<>23519232 00000100:00100000:0.0:1789760787.448992:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371392:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.522718:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973061 gen 3 00000008:00010000:0.0:1789760787.522772:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760787.522781:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.522793:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.522816:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760787.522845:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371392:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.522943:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5741, 5741] 00000080:00200000:1.0:1789760787.522958:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.522965:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.522980:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23519232 i_blocks 47111 00000020:00200000:1.0:1789760787.522983:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.522993:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.523027:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23519232, count: 4096 00000080:00200000:1.0:1789760787.523034:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23519232, bytes: 4096 00000080:00200000:1.0:1789760787.523044:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.523054:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.523056:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23519232, 23523328) 0 0 00000080:00200000:1.0:1789760787.523062:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.523078:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5742, 5742] 00020000:00200000:1.0:1789760787.523086:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23519232 chunk: [23519232, 23523328) 23523328, 4096 00020000:00200000:1.0:1789760787.523091:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.523110:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5742, 5742] 00000008:00010000:1.0:1789760787.523129:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.523135:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.523146:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23519232, 23523328) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.523159:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23519232=0x166e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.523219:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.523222:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23523328 00000020:00010000:1.0:1789760787.523230:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.523234:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.523237:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.523240:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.523243:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.523251:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.523258:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.523277:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760787.523447:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 23519232<>23523328 00000100:00100000:0.0:1789760787.523751:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760787.523771:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371520:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760787.598513:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760787.598529:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.598540:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.598562:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760787.598595:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371520:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.598702:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5742, 5742] 00000080:00200000:1.0:1789760787.598719:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.598726:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.598742:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23523328 i_blocks 47111 00000020:00200000:1.0:1789760787.598746:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.598756:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.598793:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23523328, count: 4096 00000080:00200000:1.0:1789760787.598800:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23523328, bytes: 4096 00000080:00200000:1.0:1789760787.598809:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.598818:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.598821:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23523328, 23527424) 0 0 00000080:00200000:1.0:1789760787.598826:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.598842:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5743, 5743] 00020000:00200000:1.0:1789760787.598850:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23523328 chunk: [23523328, 23527424) 23527424, 4096 00020000:00200000:1.0:1789760787.598855:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.598874:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5743, 5743] 00000008:00010000:1.0:1789760787.598891:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.599034:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.599050:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23523328, 23527424) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.599066:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23523328=0x166f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.599197:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.599200:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23527424 00000020:00010000:1.0:1789760787.599209:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.599213:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.599216:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.599220:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.599223:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.599231:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.599238:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.599257:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760787.599363:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23523328<>23527424 00000100:00100000:0.0:1789760787.599526:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371648:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.635361:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973063 gen 3 00000008:00010000:0.0:1789760787.635460:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760787.635469:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.635480:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.635502:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760787.635532:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371648:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.635741:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5743, 5743] 00000080:00200000:1.0:1789760787.635760:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.635766:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.635783:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23527424 i_blocks 47111 00000020:00200000:1.0:1789760787.635788:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.635798:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.635834:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23527424, count: 4096 00000080:00200000:1.0:1789760787.635842:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23527424, bytes: 4096 00000080:00200000:1.0:1789760787.635852:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.635863:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.635865:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23527424, 23531520) 0 0 00000080:00200000:1.0:1789760787.635871:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.635887:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5744, 5744] 00020000:00200000:1.0:1789760787.635906:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23527424 chunk: [23527424, 23531520) 23531520, 4096 00020000:00200000:1.0:1789760787.635910:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.635931:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5744, 5744] 00000008:00010000:1.0:1789760787.635949:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.635956:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.635967:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23527424, 23531520) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.635983:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23527424=0x1670000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.636043:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.636046:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23531520 00000020:00010000:1.0:1789760787.636053:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.636056:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.636059:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.636062:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.636065:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.636073:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.636080:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.636215:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760787.636328:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23527424<>23531520 00000100:00100000:0.0:1789760787.636568:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507371776:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.686997:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973064 gen 3 00000008:00010000:0.0:1789760787.687040:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760787.687046:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.687055:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.687076:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760787.687123:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507371776:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.687181:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5744, 5744] 00000080:00200000:1.0:1789760787.687203:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.687211:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.687227:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23531520 i_blocks 47111 00000020:00200000:1.0:1789760787.687233:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.687245:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.687282:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23531520, count: 4096 00000080:00200000:1.0:1789760787.687290:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23531520, bytes: 4096 00000080:00200000:1.0:1789760787.687300:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.687311:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.687313:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23531520, 23535616) 0 0 00000080:00200000:1.0:1789760787.687324:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.687341:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5745, 5745] 00020000:00200000:1.0:1789760787.687349:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23531520 chunk: [23531520, 23535616) 23535616, 4096 00020000:00200000:1.0:1789760787.687354:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.687414:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5745, 5745] 00000008:00010000:1.0:1789760787.687434:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.687440:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.687452:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23531520, 23535616) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.687467:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23531520=0x1671000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.687533:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.687536:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23535616 00000020:00010000:1.0:1789760787.687545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.687549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.687552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.687555:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.687558:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.687566:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.687573:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.687594:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760787.687782:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23531520<>23535616 00000100:00100000:0.0:1789760787.687874:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371904:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760787.708008:0:38:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 1/0, layout version 0 restore needed 0 00020000:00200000:2.0:1789760787.708032:0:38:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760787.708319:0:38:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 1/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000100:00100000:0.0:1789760787.739020:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973065 gen 3 00000008:00010000:0.0:1789760787.739075:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760787.739084:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.739096:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.739126:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760787.739159:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507371904:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.740722:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5745, 5745] 00000080:00200000:1.0:1789760787.740746:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.740754:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.740774:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23535616 i_blocks 47111 00000020:00200000:1.0:1789760787.740779:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.740794:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.740831:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23535616, count: 4096 00000080:00200000:1.0:1789760787.740838:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23535616, bytes: 4096 00000080:00200000:1.0:1789760787.740852:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.740861:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.740864:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23535616, 23539712) 0 0 00000080:00200000:1.0:1789760787.740869:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.740886:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5746, 5746] 00020000:00200000:1.0:1789760787.740894:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23535616 chunk: [23535616, 23539712) 23539712, 4096 00020000:00200000:1.0:1789760787.740899:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.740922:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5746, 5746] 00000008:00010000:1.0:1789760787.740941:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.740946:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.740957:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23535616, 23539712) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.740971:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23535616=0x1672000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.741036:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.741039:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23539712 00000020:00010000:1.0:1789760787.741047:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.741051:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.741054:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.741058:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.741451:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.741464:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.741472:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.741499:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760787.741777:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23535616<>23539712 00000100:00100000:0.0:1789760787.741923:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372032:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.819500:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973066 gen 3 00000008:00010000:0.0:1789760787.819548:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760787.819568:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.819579:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.819595:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760787.819620:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372032:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.820698:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5746, 5746] 00000080:00200000:1.0:1789760787.820719:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.820726:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.820743:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23539712 i_blocks 47111 00000020:00200000:1.0:1789760787.820749:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.820762:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.820797:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23539712, count: 4096 00000080:00200000:1.0:1789760787.820805:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23539712, bytes: 4096 00000080:00200000:1.0:1789760787.820819:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.820829:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.820832:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23539712, 23543808) 0 0 00000080:00200000:1.0:1789760787.820838:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.820854:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5747, 5747] 00020000:00200000:1.0:1789760787.820863:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23539712 chunk: [23539712, 23543808) 23543808, 4096 00020000:00200000:1.0:1789760787.820868:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.820887:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5747, 5747] 00000008:00010000:1.0:1789760787.820905:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.820911:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.820923:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23539712, 23543808) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.820938:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23539712=0x1673000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.821014:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.821215:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23543808 00000020:00010000:1.0:1789760787.821225:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.821229:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.821232:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.821235:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.821239:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.821248:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.821255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.821270:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760787.821417:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23539712<>23543808 00000100:00100000:0.0:1789760787.821967:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760787.821998:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372160:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.915327:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973067 gen 3 00000008:00010000:0.0:1789760787.915416:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760787.915425:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.915437:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.915470:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760787.915503:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372160:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.915679:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5747, 5747] 00000080:00200000:1.0:1789760787.915697:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.915702:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.915719:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23543808 i_blocks 47111 00000020:00200000:1.0:1789760787.915722:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.915733:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.915769:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23543808, count: 4096 00000080:00200000:1.0:1789760787.915777:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23543808, bytes: 4096 00000080:00200000:1.0:1789760787.915793:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.915803:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.915805:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23543808, 23547904) 0 0 00000080:00200000:1.0:1789760787.915810:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.915828:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5748, 5748] 00020000:00200000:1.0:1789760787.915837:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23543808 chunk: [23543808, 23547904) 23547904, 4096 00020000:00200000:1.0:1789760787.915842:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.915864:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5748, 5748] 00000008:00010000:1.0:1789760787.915883:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.915890:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.915901:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23543808, 23547904) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.915916:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23543808=0x1674000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.915973:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.915976:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23547904 00000020:00010000:1.0:1789760787.915984:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.915987:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.915990:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.915993:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.915996:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.916003:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.916021:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.916037:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760787.916136:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23543808<>23547904 00000100:00100000:0.0:1789760787.917064:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760787.917095:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507372288:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760787.958899:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973068 gen 3 00000008:00010000:0.0:1789760787.958957:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760787.958967:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760787.958979:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760787.959001:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760787.959035:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507372288:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760787.959793:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5748, 5748] 00000080:00200000:1.0:1789760787.959811:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760787.959818:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760787.959834:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23547904 i_blocks 47111 00000020:00200000:1.0:1789760787.959839:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760787.959850:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760787.959886:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23547904, count: 4096 00000080:00200000:1.0:1789760787.959895:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23547904, bytes: 4096 00000080:00200000:1.0:1789760787.959909:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760787.959918:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760787.959920:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23547904, 23552000) 0 0 00000080:00200000:1.0:1789760787.959925:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760787.959941:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5749, 5749] 00020000:00200000:1.0:1789760787.959948:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23547904 chunk: [23547904, 23552000) 23552000, 4096 00020000:00200000:1.0:1789760787.959953:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760787.959971:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5749, 5749] 00000008:00010000:1.0:1789760787.959990:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760787.959996:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760787.960008:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23547904, 23552000) append 0 append lockpos 0 00000080:00200000:1.0:1789760787.960023:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23547904=0x1675000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760787.960094:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760787.960097:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23552000 00000020:00010000:1.0:1789760787.960105:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760787.960109:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.960111:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760787.960114:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760787.960117:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760787.960126:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760787.960133:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760787.960151:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760787.960259:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23547904<>23552000 00000100:00100000:0.0:1789760787.961263:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760787.961352:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372416:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760788.024211:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760788.024228:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.024242:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.024265:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760788.024298:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372416:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.024457:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5749, 5749] 00000080:00200000:1.0:1789760788.024474:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.024480:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.024496:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23552000 i_blocks 47111 00000020:00200000:1.0:1789760788.024500:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.024510:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.024543:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23552000, count: 4096 00000080:00200000:1.0:1789760788.024619:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23552000, bytes: 4096 00000080:00200000:1.0:1789760788.024632:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.024650:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.024653:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23552000, 23556096) 0 0 00000080:00200000:1.0:1789760788.024659:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.024736:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5750, 5750] 00020000:00200000:1.0:1789760788.024746:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23552000 chunk: [23552000, 23556096) 23556096, 4096 00020000:00200000:1.0:1789760788.024750:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.024770:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5750, 5750] 00000008:00010000:1.0:1789760788.024789:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.024795:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.024807:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23552000, 23556096) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.024823:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23552000=0x1676000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.024883:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.024886:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23556096 00000020:00010000:1.0:1789760788.024895:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.024898:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.024901:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.024905:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.024908:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.025159:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.025167:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.025189:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760788.025301:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23552000<>23556096 00000100:00100000:0.0:1789760788.025496:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372544:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.059252:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973069 gen 3 00000100:00100000:0.0:1789760788.059300:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8169500 x1876699507372544/t4294973070(4294973070) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760804 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760788.059353:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760788.059360:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.059372:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.059472:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760788.059509:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507372544:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.063473:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5750, 5750] 00000080:00200000:1.0:1789760788.063518:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.063526:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.063546:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23556096 i_blocks 47111 00000020:00200000:1.0:1789760788.063561:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.063579:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.063617:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23556096, count: 4096 00000080:00200000:1.0:1789760788.063625:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23556096, bytes: 4096 00000080:00200000:1.0:1789760788.063652:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.063662:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.063664:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23556096, 23560192) 0 0 00000080:00200000:1.0:1789760788.063671:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.063687:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5751, 5751] 00020000:00200000:1.0:1789760788.063696:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23556096 chunk: [23556096, 23560192) 23560192, 4096 00020000:00200000:1.0:1789760788.063700:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.063721:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5751, 5751] 00000008:00010000:1.0:1789760788.063740:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.063745:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.063758:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23556096, 23560192) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.063773:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23556096=0x1677000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.063844:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.063847:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23560192 00000020:00010000:1.0:1789760788.063856:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.063860:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.063863:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.063867:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.063870:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.063879:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.063886:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.063919:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760788.065483:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23556096<>23560192 00000100:00100000:1.0:1789760788.065788:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507372672:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.135240:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973071 gen 3 00000008:00010000:0.0:1789760788.135307:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760788.135316:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.135326:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.135353:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760788.135431:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507372672:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760788.135562:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5751, 5751] 00000080:00200000:2.0:1789760788.135580:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.135588:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.135603:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23560192 i_blocks 47111 00000020:00200000:2.0:1789760788.135606:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.135617:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.135663:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23560192, count: 4096 00000080:00200000:2.0:1789760788.135671:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23560192, bytes: 4096 00000080:00200000:2.0:1789760788.135681:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.135691:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.135693:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23560192, 23564288) 0 0 00000080:00200000:2.0:1789760788.135706:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.135723:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5752, 5752] 00020000:00200000:2.0:1789760788.135731:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23560192 chunk: [23560192, 23564288) 23564288, 4096 00020000:00200000:2.0:1789760788.135736:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.135754:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5752, 5752] 00000008:00010000:2.0:1789760788.135772:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.135778:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.135792:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23560192, 23564288) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.135806:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23560192=0x1678000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.135865:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.135868:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23564288 00000020:00010000:2.0:1789760788.135877:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.135881:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.135885:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.135888:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.135891:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.135899:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.135905:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.135922:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760788.136205:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23560192<>23564288 00000100:00100000:2.0:1789760788.177297:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973072 gen 3 00000008:00010000:2.0:1789760788.177350:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ace40 00000020:00200000:2.0:1789760788.177359:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760788.177370:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760788.177433:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ace40 00000080:00200000:0.0:1789760788.177569:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5752, 5752] 00000080:00200000:0.0:1789760788.177590:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760788.177596:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760788.177612:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23564288 i_blocks 47111 00000020:00200000:0.0:1789760788.177616:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760788.177627:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760788.177675:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23564288, count: 4096 00000080:00200000:0.0:1789760788.177684:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23564288, bytes: 4096 00000080:00200000:0.0:1789760788.177694:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760788.177703:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760788.177705:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23564288, 23568384) 0 0 00000080:00200000:0.0:1789760788.177710:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760788.177728:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5753, 5753] 00020000:00200000:0.0:1789760788.177737:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23564288 chunk: [23564288, 23568384) 23568384, 4096 00020000:00200000:0.0:1789760788.177743:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760788.177762:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5753, 5753] 00000008:00010000:0.0:1789760788.177782:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760788.177788:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760788.177800:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23564288, 23568384) append 0 append lockpos 0 00000080:00200000:0.0:1789760788.177815:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23564288=0x1679000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760788.177870:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760788.177873:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23568384 00000020:00010000:0.0:1789760788.177881:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760788.177885:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.177888:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.177892:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760788.177905:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760788.177915:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760788.177923:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000100:00100000:2.0:1789760788.178025:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507372800:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760788.178073:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef76c0 00000008:00100000:2.0:1789760788.178181:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23564288<>23568384 00000100:00100000:2.0:1789760788.178235:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507372928:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760788.221537:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2419:1876699507373056:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760788.221666:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507373312:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760788.221699:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2419:1876699507373568:192.168.203.141@tcp:400:kworker.0 00000080:00200000:0.0:1789760788.227993:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5753, 5753] 00000080:00200000:0.0:1789760788.228024:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760788.228032:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760788.228050:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23568384 i_blocks 47111 00000020:00200000:0.0:1789760788.228056:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760788.228073:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760788.228111:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23568384, count: 4096 00000080:00200000:0.0:1789760788.228120:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23568384, bytes: 4096 00000080:00200000:0.0:1789760788.228135:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760788.228146:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760788.228148:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23568384, 23572480) 0 0 00000080:00200000:0.0:1789760788.228153:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760788.228181:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5754, 5754] 00020000:00200000:0.0:1789760788.228190:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23568384 chunk: [23568384, 23572480) 23572480, 4096 00020000:00200000:0.0:1789760788.228202:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760788.228226:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5754, 5754] 00000008:00010000:0.0:1789760788.228245:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760788.228251:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760788.228263:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23568384, 23572480) append 0 append lockpos 0 00000080:00200000:0.0:1789760788.228278:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23568384=0x167a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760788.228348:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760788.228351:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23572480 00000020:00010000:0.0:1789760788.228360:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760788.228363:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.228367:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.228371:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760788.228374:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760788.228417:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760788.228424:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760788.228454:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760788.228564:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23568384<>23572480 00000100:00100000:0.0:1789760788.228611:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760788.228648:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507373824:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760788.229963:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2419:1876699507373056:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760788.229979:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507373312:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760788.229991:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2419:1876699507373568:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760788.275565:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973074 gen 3 00000008:00010000:0.0:1789760788.275623:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760788.275633:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.275661:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.275694:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760788.275728:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507373824:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760788.275860:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5754, 5754] 00000080:00200000:2.0:1789760788.275877:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.275884:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.275901:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23572480 i_blocks 47111 00000020:00200000:2.0:1789760788.275905:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.275915:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.275953:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23572480, count: 4096 00000080:00200000:2.0:1789760788.275960:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23572480, bytes: 4096 00000080:00200000:2.0:1789760788.275976:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.275985:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.275987:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23572480, 23576576) 0 0 00000080:00200000:2.0:1789760788.275992:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.276008:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5755, 5755] 00020000:00200000:2.0:1789760788.276016:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23572480 chunk: [23572480, 23576576) 23576576, 4096 00020000:00200000:2.0:1789760788.276021:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.276041:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5755, 5755] 00000008:00010000:2.0:1789760788.276061:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.276068:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.276080:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23572480, 23576576) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.276094:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23572480=0x167b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.276154:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.276157:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23576576 00000020:00010000:2.0:1789760788.276166:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.276170:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.276173:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.276177:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.276180:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.276188:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.276208:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.276225:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760788.276424:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 23572480<>23576576 00000080:00200000:2.0:1789760788.315456:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5755, 5755] 00000080:00200000:2.0:1789760788.315483:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.315489:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.315506:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23576576 i_blocks 47111 00000020:00200000:2.0:1789760788.315510:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.315522:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.315559:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23576576, count: 4096 00000080:00200000:2.0:1789760788.315567:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23576576, bytes: 4096 00000080:00200000:2.0:1789760788.315576:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.315586:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.315588:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23576576, 23580672) 0 0 00000080:00200000:2.0:1789760788.315593:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.315611:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5756, 5756] 00020000:00200000:2.0:1789760788.315620:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23576576 chunk: [23576576, 23580672) 23580672, 4096 00020000:00200000:2.0:1789760788.315626:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.315659:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5756, 5756] 00000008:00010000:2.0:1789760788.315679:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.315685:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.315816:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23576576, 23580672) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.315835:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23576576=0x167c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.315910:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.315913:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23580672 00000020:00010000:2.0:1789760788.315923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.315926:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.315930:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.315934:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.315937:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.315944:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.315953:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.315974:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760788.316154:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 23576576<>23580672 00000080:00200000:2.0:1789760788.353437:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5756, 5756] 00000080:00200000:2.0:1789760788.353455:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.353461:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.353478:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23580672 i_blocks 47111 00000020:00200000:2.0:1789760788.353482:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.353492:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.353526:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23580672, count: 4096 00000080:00200000:2.0:1789760788.353533:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23580672, bytes: 4096 00000080:00200000:2.0:1789760788.353542:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.353552:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.353554:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23580672, 23584768) 0 0 00000080:00200000:2.0:1789760788.353558:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.353574:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5757, 5757] 00020000:00200000:2.0:1789760788.353582:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23580672 chunk: [23580672, 23584768) 23584768, 4096 00020000:00200000:2.0:1789760788.353587:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.353606:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5757, 5757] 00000008:00010000:2.0:1789760788.353623:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.353628:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.353647:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23580672, 23584768) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.353663:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23580672=0x167d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.353777:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.353780:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23584768 00000020:00010000:2.0:1789760788.353789:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.353792:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.353795:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.353800:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.353802:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.353809:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.353816:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.353831:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760788.353934:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 23580672<>23584768 00000080:00200000:2.0:1789760788.391472:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5757, 5757] 00000080:00200000:2.0:1789760788.391492:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.391499:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.391517:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23584768 i_blocks 47111 00000020:00200000:2.0:1789760788.391522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.391532:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.391564:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23584768, count: 4096 00000080:00200000:2.0:1789760788.391571:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23584768, bytes: 4096 00000080:00200000:2.0:1789760788.391585:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.391593:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.391595:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23584768, 23588864) 0 0 00000080:00200000:2.0:1789760788.391600:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.391615:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5758, 5758] 00020000:00200000:2.0:1789760788.391623:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23584768 chunk: [23584768, 23588864) 23588864, 4096 00020000:00200000:2.0:1789760788.391627:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.391656:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5758, 5758] 00000008:00010000:2.0:1789760788.391682:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.391688:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.391706:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23584768, 23588864) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.391721:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23584768=0x167e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.391779:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.391782:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23588864 00000020:00010000:2.0:1789760788.391790:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.391794:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.391797:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.391802:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.391805:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.391812:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.391819:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.391838:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760788.391941:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 23584768<>23588864 00000080:00200000:2.0:1789760788.478333:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5758, 5758] 00000080:00200000:2.0:1789760788.478352:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.478358:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.478408:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23588864 i_blocks 47111 00000020:00200000:2.0:1789760788.478412:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.478424:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.478532:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23588864, count: 4096 00000080:00200000:2.0:1789760788.478541:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23588864, bytes: 4096 00000080:00200000:2.0:1789760788.478556:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.478565:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.478568:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23588864, 23592960) 0 0 00000080:00200000:2.0:1789760788.478573:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.478591:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5759, 5759] 00020000:00200000:2.0:1789760788.478599:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23588864 chunk: [23588864, 23592960) 23592960, 4096 00020000:00200000:2.0:1789760788.478604:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.478626:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5759, 5759] 00000008:00010000:2.0:1789760788.478653:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.478660:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.478671:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23588864, 23592960) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.478687:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23588864=0x167f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.478748:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.478751:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23592960 00000020:00010000:2.0:1789760788.478759:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.478763:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.478766:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.478770:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.478774:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.478782:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.478789:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.478807:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760788.478922:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 23588864<>23592960 00000100:00100000:2.0:1789760788.478969:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760788.478988:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507374464:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760788.526486:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973079 gen 3 00000008:00010000:2.0:1789760788.526627:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:2.0:1789760788.526645:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760788.526657:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760788.526681:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:2.0:1789760788.526710:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507374464:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760788.526820:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5759, 5759] 00000080:00200000:0.0:1789760788.526835:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760788.526839:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760788.526853:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23592960 i_blocks 47111 00000020:00200000:0.0:1789760788.526856:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760788.526865:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760788.526902:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23592960, count: 4096 00000080:00200000:0.0:1789760788.526908:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23592960, bytes: 4096 00000080:00200000:0.0:1789760788.526917:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760788.526926:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760788.526927:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23592960, 23597056) 0 0 00000080:00200000:0.0:1789760788.526931:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760788.527036:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5760, 5760] 00020000:00200000:0.0:1789760788.527044:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23592960 chunk: [23592960, 23597056) 23597056, 4096 00020000:00200000:0.0:1789760788.527049:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760788.527070:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5760, 5760] 00000008:00010000:0.0:1789760788.527207:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760788.527214:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760788.527227:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23592960, 23597056) append 0 append lockpos 0 00000080:00200000:0.0:1789760788.527242:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23592960=0x1680000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760788.527299:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760788.527302:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23597056 00000020:00010000:0.0:1789760788.527310:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760788.527314:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.527317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.527320:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760788.527323:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760788.527332:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760788.527339:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760788.527358:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760788.528324:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23592960<>23597056 00000100:00100000:0.0:1789760788.528423:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507374592:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.570240:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973080 gen 3 00000008:00010000:0.0:1789760788.570291:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760788.570300:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.570313:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.570338:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760788.570375:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507374592:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.570887:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5760, 5760] 00000080:00200000:1.0:1789760788.570907:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.570914:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.570939:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23597056 i_blocks 47111 00000020:00200000:1.0:1789760788.570944:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.570953:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.570988:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23597056, count: 4096 00000080:00200000:1.0:1789760788.570996:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23597056, bytes: 4096 00000080:00200000:1.0:1789760788.571008:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.571018:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.571021:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23597056, 23601152) 0 0 00000080:00200000:1.0:1789760788.571025:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.571041:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5761, 5761] 00020000:00200000:1.0:1789760788.571050:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23597056 chunk: [23597056, 23601152) 23601152, 4096 00020000:00200000:1.0:1789760788.571055:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.571074:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5761, 5761] 00000008:00010000:1.0:1789760788.571103:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.571109:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.571121:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23597056, 23601152) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.571136:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23597056=0x1681000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.571196:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.571198:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23601152 00000020:00010000:1.0:1789760788.571207:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.571210:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.571217:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.571221:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.571224:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.571232:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.571239:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.571257:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760788.571362:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23597056<>23601152 00000100:00100000:0.0:1789760788.571704:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507374720:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760788.636432:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760788.636452:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.636464:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.636484:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760788.636516:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507374720:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.636666:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5761, 5761] 00000080:00200000:1.0:1789760788.636685:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.636692:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.636953:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23601152 i_blocks 47111 00000020:00200000:1.0:1789760788.636959:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.636969:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.637006:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23601152, count: 4096 00000080:00200000:1.0:1789760788.637014:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23601152, bytes: 4096 00000080:00200000:1.0:1789760788.637025:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.637035:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.637037:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23601152, 23605248) 0 0 00000080:00200000:1.0:1789760788.637042:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.637059:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5762, 5762] 00020000:00200000:1.0:1789760788.637069:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23601152 chunk: [23601152, 23605248) 23605248, 4096 00020000:00200000:1.0:1789760788.637074:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.637093:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5762, 5762] 00000008:00010000:1.0:1789760788.637112:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.637118:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.637130:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23601152, 23605248) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.637146:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23601152=0x1682000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.637210:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.637213:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23605248 00000020:00010000:1.0:1789760788.637221:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.637225:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.637228:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.637232:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.637235:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.637244:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.637252:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.637270:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760788.638034:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23601152<>23605248 00000100:00100000:0.0:1789760788.638161:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507374848:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.682655:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973082 gen 3 00000008:00010000:0.0:1789760788.682739:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760788.682747:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.682757:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.682781:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760788.682820:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507374848:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.683140:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5762, 5762] 00000080:00200000:1.0:1789760788.683157:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.683164:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.683181:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23605248 i_blocks 47111 00000020:00200000:1.0:1789760788.683185:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.683197:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.683235:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23605248, count: 4096 00000080:00200000:1.0:1789760788.683242:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23605248, bytes: 4096 00000080:00200000:1.0:1789760788.683257:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.683268:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.683270:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23605248, 23609344) 0 0 00000080:00200000:1.0:1789760788.683274:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.683290:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5763, 5763] 00020000:00200000:1.0:1789760788.683299:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23605248 chunk: [23605248, 23609344) 23609344, 4096 00020000:00200000:1.0:1789760788.683303:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.683324:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5763, 5763] 00000008:00010000:1.0:1789760788.683343:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.683349:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.683362:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23605248, 23609344) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.683421:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23605248=0x1683000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.683485:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.683489:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23609344 00000020:00010000:1.0:1789760788.683497:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.683502:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.683505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.683510:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.683514:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.683523:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.683531:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.683550:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760788.683812:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23605248<>23609344 00000100:00100000:0.0:1789760788.684101:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507374976:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.754595:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973083 gen 3 00000008:00010000:0.0:1789760788.754667:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760788.754675:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.754686:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.754708:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760788.754740:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507374976:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.754921:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5763, 5763] 00000080:00200000:1.0:1789760788.754939:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.754946:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.754974:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23609344 i_blocks 47111 00000020:00200000:1.0:1789760788.754979:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.754989:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.755030:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23609344, count: 4096 00000080:00200000:1.0:1789760788.755038:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23609344, bytes: 4096 00000080:00200000:1.0:1789760788.755049:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.755059:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.755061:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23609344, 23613440) 0 0 00000080:00200000:1.0:1789760788.755066:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.755217:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5764, 5764] 00020000:00200000:1.0:1789760788.755229:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23609344 chunk: [23609344, 23613440) 23613440, 4096 00020000:00200000:1.0:1789760788.755234:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.755259:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5764, 5764] 00000008:00010000:1.0:1789760788.755282:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.755288:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.755302:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23609344, 23613440) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.755318:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23609344=0x1684000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.755570:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.755574:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23613440 00000020:00010000:1.0:1789760788.755584:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.755588:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.755596:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.755600:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.755603:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.755613:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.755902:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.755929:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760788.756046:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 23609344<>23613440 00000100:00100000:0.0:1789760788.756237:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507375104:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760788.781299:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760788.781316:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.781328:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.781348:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760788.781493:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507375104:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.781591:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5764, 5764] 00000080:00200000:1.0:1789760788.781608:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.781614:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.781643:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23613440 i_blocks 47111 00000020:00200000:1.0:1789760788.781647:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.781656:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.781690:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23613440, count: 4096 00000080:00200000:1.0:1789760788.781696:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23613440, bytes: 4096 00000080:00200000:1.0:1789760788.781706:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.781716:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.781719:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23613440, 23617536) 0 0 00000080:00200000:1.0:1789760788.781724:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.781741:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5765, 5765] 00020000:00200000:1.0:1789760788.781750:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23613440 chunk: [23613440, 23617536) 23617536, 4096 00020000:00200000:1.0:1789760788.781754:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.781773:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5765, 5765] 00000008:00010000:1.0:1789760788.781790:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.781797:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.781807:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23613440, 23617536) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.781822:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23613440=0x1685000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.781885:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.781889:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23617536 00000020:00010000:1.0:1789760788.781897:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.781901:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.781904:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.781908:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.781911:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.781919:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.781926:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.782004:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760788.782111:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23613440<>23617536 00000100:00100000:0.0:1789760788.782232:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507375232:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.818184:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973085 gen 3 00000008:00010000:0.0:1789760788.818255:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760788.818264:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.818275:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.818303:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760788.818336:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507375232:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760788.818480:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5765, 5765] 00000080:00200000:2.0:1789760788.818497:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.818504:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.818519:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23617536 i_blocks 47111 00000020:00200000:2.0:1789760788.818523:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.818534:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.818570:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23617536, count: 4096 00000080:00200000:2.0:1789760788.818578:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23617536, bytes: 4096 00000080:00200000:2.0:1789760788.818588:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.818596:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.818599:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23617536, 23621632) 0 0 00000080:00200000:2.0:1789760788.818603:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.818620:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5766, 5766] 00020000:00200000:2.0:1789760788.818629:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23617536 chunk: [23617536, 23621632) 23621632, 4096 00020000:00200000:2.0:1789760788.818645:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.818664:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5766, 5766] 00000008:00010000:2.0:1789760788.818751:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.818757:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.818769:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23617536, 23621632) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.818783:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23617536=0x1686000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.818838:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.818841:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23621632 00000020:00010000:2.0:1789760788.818851:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.818854:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.818857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.818860:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.818863:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.818870:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.818877:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.818893:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760788.819057:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 23617536<>23621632 00000080:00200000:0.0:1789760788.848368:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5766, 5766] 00000080:00200000:0.0:1789760788.848467:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760788.848473:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760788.848490:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23621632 i_blocks 47111 00000020:00200000:0.0:1789760788.848493:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760788.848503:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760788.848538:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23621632, count: 4096 00000080:00200000:0.0:1789760788.848545:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23621632, bytes: 4096 00000080:00200000:0.0:1789760788.848556:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760788.848565:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760788.848567:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23621632, 23625728) 0 0 00000080:00200000:0.0:1789760788.848572:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760788.848589:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5767, 5767] 00020000:00200000:0.0:1789760788.848596:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23621632 chunk: [23621632, 23625728) 23625728, 4096 00020000:00200000:0.0:1789760788.848601:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760788.848619:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5767, 5767] 00000008:00010000:0.0:1789760788.848647:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760788.848653:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760788.848663:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23621632, 23625728) append 0 append lockpos 0 00000080:00200000:0.0:1789760788.848732:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23621632=0x1687000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760788.848786:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760788.848788:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23625728 00000020:00010000:0.0:1789760788.848797:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760788.848800:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.848804:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760788.848807:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760788.848810:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760788.848817:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760788.848823:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760788.848838:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760788.849199:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23621632<>23625728 00000100:00100000:1.0:1789760788.849273:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507375488:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.876262:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973087 gen 3 00000008:00010000:0.0:1789760788.876319:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760788.876328:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.876340:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.876453:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760788.876484:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507375488:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760788.876609:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5767, 5767] 00000080:00200000:2.0:1789760788.876626:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760788.876641:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760788.876658:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23625728 i_blocks 47111 00000020:00200000:2.0:1789760788.876662:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760788.876725:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760788.876761:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23625728, count: 4096 00000080:00200000:2.0:1789760788.876769:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23625728, bytes: 4096 00000080:00200000:2.0:1789760788.876778:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760788.876787:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760788.876790:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23625728, 23629824) 0 0 00000080:00200000:2.0:1789760788.876794:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760788.876811:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5768, 5768] 00020000:00200000:2.0:1789760788.876820:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23625728 chunk: [23625728, 23629824) 23629824, 4096 00020000:00200000:2.0:1789760788.876824:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760788.876844:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5768, 5768] 00000008:00010000:2.0:1789760788.876862:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760788.876867:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760788.876879:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23625728, 23629824) append 0 append lockpos 0 00000080:00200000:2.0:1789760788.876893:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23625728=0x1688000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760788.876993:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760788.876995:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23629824 00000020:00010000:2.0:1789760788.877003:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760788.877006:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.877009:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760788.877013:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760788.877015:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760788.877023:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760788.877031:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760788.877048:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760788.877260:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 23625728<>23629824 00000080:00200000:1.0:1789760788.908188:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5768, 5768] 00000080:00200000:1.0:1789760788.908202:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.908207:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.908220:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23629824 i_blocks 47111 00000020:00200000:1.0:1789760788.908223:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.908232:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.908262:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23629824, count: 4096 00000080:00200000:1.0:1789760788.908268:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23629824, bytes: 4096 00000080:00200000:1.0:1789760788.908277:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.908284:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.908286:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23629824, 23633920) 0 0 00000080:00200000:1.0:1789760788.908289:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.908305:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5769, 5769] 00020000:00200000:1.0:1789760788.908314:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23629824 chunk: [23629824, 23633920) 23633920, 4096 00020000:00200000:1.0:1789760788.908319:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.908339:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5769, 5769] 00000008:00010000:1.0:1789760788.908357:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.908363:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.908406:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23629824, 23633920) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.908421:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23629824=0x1689000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.908476:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.908479:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23633920 00000020:00010000:1.0:1789760788.908487:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.908490:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.908493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.908497:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.908499:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.908507:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.908512:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.908528:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760788.908628:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23629824<>23633920 00000100:00100000:0.0:1789760788.908718:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507375744:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760788.943928:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760788.943946:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.943959:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.943981:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760788.944012:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507375744:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.944133:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5769, 5769] 00000080:00200000:1.0:1789760788.944148:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.944155:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.944171:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23633920 i_blocks 47111 00000020:00200000:1.0:1789760788.944175:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.944184:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.944217:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23633920, count: 4096 00000080:00200000:1.0:1789760788.944226:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23633920, bytes: 4096 00000080:00200000:1.0:1789760788.944235:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.944244:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.944246:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23633920, 23638016) 0 0 00000080:00200000:1.0:1789760788.944251:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.944268:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5770, 5770] 00020000:00200000:1.0:1789760788.944278:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23633920 chunk: [23633920, 23638016) 23638016, 4096 00020000:00200000:1.0:1789760788.944283:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.944302:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5770, 5770] 00000008:00010000:1.0:1789760788.944321:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.944326:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.944338:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23633920, 23638016) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.944352:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23633920=0x168a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.944492:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.944495:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23638016 00000020:00010000:1.0:1789760788.944505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.944508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.944511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.944515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.944517:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.944525:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.944532:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.944551:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760788.944758:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23633920<>23638016 00000100:00100000:0.0:1789760788.944850:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760788.944869:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507375872:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760788.988176:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973090 gen 3 00000008:00010000:0.0:1789760788.988244:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760788.988253:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760788.988263:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760788.988283:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760788.988313:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507375872:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760788.988498:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5770, 5770] 00000080:00200000:1.0:1789760788.988515:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760788.988522:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760788.988548:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23638016 i_blocks 47111 00000020:00200000:1.0:1789760788.988552:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760788.988562:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760788.988596:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23638016, count: 4096 00000080:00200000:1.0:1789760788.988604:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23638016, bytes: 4096 00000080:00200000:1.0:1789760788.988614:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760788.988623:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760788.988625:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23638016, 23642112) 0 0 00000080:00200000:1.0:1789760788.988639:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760788.988657:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5771, 5771] 00020000:00200000:1.0:1789760788.988666:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23638016 chunk: [23638016, 23642112) 23642112, 4096 00020000:00200000:1.0:1789760788.988674:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760788.988696:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5771, 5771] 00000008:00010000:1.0:1789760788.988715:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760788.988721:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760788.988733:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23638016, 23642112) append 0 append lockpos 0 00000080:00200000:1.0:1789760788.988747:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23638016=0x168b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760788.988808:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760788.988811:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23642112 00000020:00010000:1.0:1789760788.988820:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760788.988824:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.988827:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760788.988831:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760788.988834:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760788.988843:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760788.988850:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760788.988869:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760788.988977:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23638016<>23642112 00000100:00100000:0.0:1789760788.989045:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507376000:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.042039:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973091 gen 3 00000008:00010000:0.0:1789760789.042093:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760789.042103:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.042115:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.042139:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760789.042171:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507376000:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760789.042282:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5771, 5771] 00000080:00200000:1.0:1789760789.042298:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.042304:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.042321:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23642112 i_blocks 47111 00000020:00200000:1.0:1789760789.042325:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.042335:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.042366:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23642112, count: 4096 00000080:00200000:1.0:1789760789.042373:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23642112, bytes: 4096 00000080:00200000:1.0:1789760789.042413:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.042422:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.042424:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23642112, 23646208) 0 0 00000080:00200000:1.0:1789760789.042429:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.042447:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5772, 5772] 00020000:00200000:1.0:1789760789.042455:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23642112 chunk: [23642112, 23646208) 23646208, 4096 00020000:00200000:1.0:1789760789.042460:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.042481:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5772, 5772] 00000008:00010000:1.0:1789760789.042500:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.042507:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.042519:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23642112, 23646208) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.042537:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23642112=0x168c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.042596:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.042599:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23646208 00000020:00010000:1.0:1789760789.042607:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.042611:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.042614:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.042618:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.042621:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.042639:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.042646:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.042663:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760789.042773:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23642112<>23646208 00000100:00100000:0.0:1789760789.043158:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507376128:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.122900:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973092 gen 3 00000008:00010000:0.0:1789760789.123015:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760789.123024:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.123035:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.123126:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760789.123157:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507376128:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760789.123486:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5772, 5772] 00000080:00200000:1.0:1789760789.123503:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.123510:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.123526:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23646208 i_blocks 47111 00000020:00200000:1.0:1789760789.123530:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.123540:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.123577:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23646208, count: 4096 00000080:00200000:1.0:1789760789.123585:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23646208, bytes: 4096 00000080:00200000:1.0:1789760789.123599:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.123608:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.123610:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23646208, 23650304) 0 0 00000080:00200000:1.0:1789760789.123615:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.123750:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5773, 5773] 00020000:00200000:1.0:1789760789.123760:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23646208 chunk: [23646208, 23650304) 23650304, 4096 00020000:00200000:1.0:1789760789.123765:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.123787:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5773, 5773] 00000008:00010000:1.0:1789760789.123806:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.123813:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.123826:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23646208, 23650304) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.123841:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23646208=0x168d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.123907:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.123910:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23650304 00000020:00010000:1.0:1789760789.123919:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.123923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.123926:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.123929:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.123932:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.126604:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.126625:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.126662:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760789.127575:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23646208<>23650304 00000100:00100000:1.0:1789760789.127649:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760789.127963:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507376256:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.169120:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973093 gen 3 00000008:00010000:0.0:1789760789.169178:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760789.169311:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.169324:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.169365:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760789.169434:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507376256:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760789.169975:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5773, 5773] 00000080:00200000:2.0:1789760789.169992:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.169998:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.170014:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23650304 i_blocks 47111 00000020:00200000:2.0:1789760789.170018:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.170029:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.170064:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23650304, count: 4096 00000080:00200000:2.0:1789760789.170141:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23650304, bytes: 4096 00000080:00200000:2.0:1789760789.170159:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.170169:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.170171:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23650304, 23654400) 0 0 00000080:00200000:2.0:1789760789.170176:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.170260:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5774, 5774] 00020000:00200000:2.0:1789760789.170270:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23650304 chunk: [23650304, 23654400) 23654400, 4096 00020000:00200000:2.0:1789760789.170275:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.170297:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5774, 5774] 00000008:00010000:2.0:1789760789.170318:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.170324:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.170337:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23650304, 23654400) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.170481:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23650304=0x168e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.170642:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.170645:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23654400 00000020:00010000:2.0:1789760789.170655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.170659:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.170663:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.170666:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.170736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.170746:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.170753:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.170775:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760789.170882:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 23650304<>23654400 00000080:00200000:1.0:1789760789.230683:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5774, 5774] 00000080:00200000:1.0:1789760789.230703:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.230709:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.230726:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23654400 i_blocks 47111 00000020:00200000:1.0:1789760789.230732:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.230742:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.230777:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23654400, count: 4096 00000080:00200000:1.0:1789760789.230785:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23654400, bytes: 4096 00000080:00200000:1.0:1789760789.230796:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.230805:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.230807:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23654400, 23658496) 0 0 00000080:00200000:1.0:1789760789.230812:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.230828:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5775, 5775] 00020000:00200000:1.0:1789760789.230836:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23654400 chunk: [23654400, 23658496) 23658496, 4096 00020000:00200000:1.0:1789760789.230841:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.230862:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5775, 5775] 00000008:00010000:1.0:1789760789.230882:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.230888:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.230899:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23654400, 23658496) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.230913:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23654400=0x168f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.230970:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.230973:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23658496 00000020:00010000:1.0:1789760789.230981:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.230985:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.230988:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.230991:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.230994:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.231002:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.231008:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.231023:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760789.231133:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23654400<>23658496 00000100:00100000:0.0:1789760789.231218:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507376512:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.266978:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973095 gen 3 00000008:00010000:0.0:1789760789.267030:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760789.267038:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.267047:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.267178:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760789.267215:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507376512:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760789.267340:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5775, 5775] 00000080:00200000:2.0:1789760789.267355:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.267360:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.267392:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23658496 i_blocks 47111 00000020:00200000:2.0:1789760789.267396:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.267408:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.267442:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23658496, count: 4096 00000080:00200000:2.0:1789760789.267449:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23658496, bytes: 4096 00000080:00200000:2.0:1789760789.267463:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.267471:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.267474:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23658496, 23662592) 0 0 00000080:00200000:2.0:1789760789.267478:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.267495:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5776, 5776] 00020000:00200000:2.0:1789760789.267503:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23658496 chunk: [23658496, 23662592) 23662592, 4096 00020000:00200000:2.0:1789760789.267508:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.267526:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5776, 5776] 00000008:00010000:2.0:1789760789.267545:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.267551:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.267562:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23658496, 23662592) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.267575:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23658496=0x1690000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.267646:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.267649:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23662592 00000020:00010000:2.0:1789760789.267656:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.267660:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.267764:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.267769:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.267772:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.267780:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.267788:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.267806:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760789.267998:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 23658496<>23662592 00000080:00200000:2.0:1789760789.308845:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5776, 5776] 00000080:00200000:2.0:1789760789.308862:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.308868:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.308884:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23662592 i_blocks 47111 00000020:00200000:2.0:1789760789.308887:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.308897:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.308933:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23662592, count: 4096 00000080:00200000:2.0:1789760789.308939:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23662592, bytes: 4096 00000080:00200000:2.0:1789760789.308950:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.308958:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.308961:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23662592, 23666688) 0 0 00000080:00200000:2.0:1789760789.308965:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.308980:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5777, 5777] 00020000:00200000:2.0:1789760789.308989:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23662592 chunk: [23662592, 23666688) 23666688, 4096 00020000:00200000:2.0:1789760789.308993:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.309013:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5777, 5777] 00000008:00010000:2.0:1789760789.309032:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.309038:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.309050:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23662592, 23666688) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.309065:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23662592=0x1691000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.310818:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.310829:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23666688 00000020:00010000:2.0:1789760789.310856:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.310861:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.310866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.310870:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.310873:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.310887:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.310897:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.310923:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760789.311042:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 23662592<>23666688 00010000:00010000:0.0:1789760789.311800:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1789760789.355543:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5777, 5777] 00000080:00200000:2.0:1789760789.355560:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.355566:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.355582:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23666688 i_blocks 47111 00000020:00200000:2.0:1789760789.355585:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.355596:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.355645:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23666688, count: 4096 00000080:00200000:2.0:1789760789.355652:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23666688, bytes: 4096 00000080:00200000:2.0:1789760789.355671:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.355681:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.355684:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23666688, 23670784) 0 0 00000080:00200000:2.0:1789760789.355688:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.355704:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5778, 5778] 00020000:00200000:2.0:1789760789.355713:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23666688 chunk: [23666688, 23670784) 23670784, 4096 00020000:00200000:2.0:1789760789.355718:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.355739:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5778, 5778] 00000008:00010000:2.0:1789760789.355757:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.355767:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.355779:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23666688, 23670784) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.355793:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23666688=0x1692000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.355854:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.355857:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23670784 00000020:00010000:2.0:1789760789.355866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.355869:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.355872:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.355876:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.355878:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.355886:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.355892:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.355923:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760789.356029:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 23666688<>23670784 00000080:00200000:2.0:1789760789.388638:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5778, 5778] 00000080:00200000:2.0:1789760789.388654:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.388660:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.388734:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23670784 i_blocks 47111 00000020:00200000:2.0:1789760789.388739:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.388749:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.388784:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23670784, count: 4096 00000080:00200000:2.0:1789760789.388792:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23670784, bytes: 4096 00000080:00200000:2.0:1789760789.388801:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.388811:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.388813:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23670784, 23674880) 0 0 00000080:00200000:2.0:1789760789.388818:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.388834:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5779, 5779] 00020000:00200000:2.0:1789760789.388842:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23670784 chunk: [23670784, 23674880) 23674880, 4096 00020000:00200000:2.0:1789760789.388847:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.388867:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5779, 5779] 00000008:00010000:2.0:1789760789.388886:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.388892:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.388973:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23670784, 23674880) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.388989:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23670784=0x1693000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.389049:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.389052:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23674880 00000020:00010000:2.0:1789760789.389061:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.389065:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.389068:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.389182:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.389187:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.389196:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.389203:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.389222:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760789.389323:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 23670784<>23674880 00000080:00200000:0.0:1789760789.423333:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5779, 5779] 00000080:00200000:0.0:1789760789.423351:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760789.423357:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760789.423373:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23674880 i_blocks 47111 00000020:00200000:0.0:1789760789.423407:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760789.423420:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760789.423455:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23674880, count: 4096 00000080:00200000:0.0:1789760789.423462:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23674880, bytes: 4096 00000080:00200000:0.0:1789760789.423474:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760789.423484:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760789.423486:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23674880, 23678976) 0 0 00000080:00200000:0.0:1789760789.423491:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760789.423510:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5780, 5780] 00020000:00200000:0.0:1789760789.423519:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23674880 chunk: [23674880, 23678976) 23678976, 4096 00020000:00200000:0.0:1789760789.423524:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760789.423544:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5780, 5780] 00000008:00010000:0.0:1789760789.423563:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760789.423569:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760789.423582:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23674880, 23678976) append 0 append lockpos 0 00000080:00200000:0.0:1789760789.423597:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23674880=0x1694000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760789.423781:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760789.423785:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23678976 00000020:00010000:0.0:1789760789.423795:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760789.423799:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760789.423802:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760789.423806:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760789.423809:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760789.423818:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760789.423826:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760789.423848:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760789.424030:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23674880<>23678976 00000100:00100000:1.0:1789760789.424177:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507377152:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.460354:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973100 gen 3 00000008:00010000:0.0:1789760789.460448:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760789.460456:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.460468:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.460492:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760789.460522:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507377152:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760789.461232:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5780, 5780] 00000080:00200000:2.0:1789760789.461252:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.461260:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.461277:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23678976 i_blocks 47111 00000020:00200000:2.0:1789760789.461281:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.461293:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.461325:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23678976, count: 4096 00000080:00200000:2.0:1789760789.461332:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23678976, bytes: 4096 00000080:00200000:2.0:1789760789.461344:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.461353:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.461355:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23678976, 23683072) 0 0 00000080:00200000:2.0:1789760789.461360:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.461404:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5781, 5781] 00020000:00200000:2.0:1789760789.461413:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23678976 chunk: [23678976, 23683072) 23683072, 4096 00020000:00200000:2.0:1789760789.461417:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.461441:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5781, 5781] 00000008:00010000:2.0:1789760789.461464:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.461471:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.461482:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23678976, 23683072) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.461498:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23678976=0x1695000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.461755:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.461758:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23683072 00000020:00010000:2.0:1789760789.461768:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.461772:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.461775:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.461778:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.461781:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.461790:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.461797:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.461816:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760789.461997:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 23678976<>23683072 00000080:00200000:2.0:1789760789.513211:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5781, 5781] 00000080:00200000:2.0:1789760789.513227:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.513234:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.513250:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23683072 i_blocks 47111 00000020:00200000:2.0:1789760789.513254:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.513264:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.513298:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23683072, count: 4096 00000080:00200000:2.0:1789760789.513306:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23683072, bytes: 4096 00000080:00200000:2.0:1789760789.513315:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.513324:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.513326:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23683072, 23687168) 0 0 00000080:00200000:2.0:1789760789.513331:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.513348:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5782, 5782] 00020000:00200000:2.0:1789760789.513356:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23683072 chunk: [23683072, 23687168) 23687168, 4096 00020000:00200000:2.0:1789760789.513360:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.513419:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5782, 5782] 00000008:00010000:2.0:1789760789.513440:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.513446:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.513458:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23683072, 23687168) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.513472:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23683072=0x1696000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.513531:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.513534:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23687168 00000020:00010000:2.0:1789760789.513541:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.513545:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.513548:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.513552:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.513555:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.513563:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.513569:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.513587:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760789.513762:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 23683072<>23687168 00000080:00200000:1.0:1789760789.594252:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5782, 5782] 00000080:00200000:1.0:1789760789.594269:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.594276:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.594290:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23687168 i_blocks 47111 00000020:00200000:1.0:1789760789.594294:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.594303:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.594337:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23687168, count: 4096 00000080:00200000:1.0:1789760789.594344:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23687168, bytes: 4096 00000080:00200000:1.0:1789760789.594355:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.594365:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.594367:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23687168, 23691264) 0 0 00000080:00200000:1.0:1789760789.594371:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.594471:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5783, 5783] 00020000:00200000:1.0:1789760789.594479:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23687168 chunk: [23687168, 23691264) 23691264, 4096 00020000:00200000:1.0:1789760789.594484:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.594504:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5783, 5783] 00000008:00010000:1.0:1789760789.594521:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.594527:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.594538:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23687168, 23691264) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.594552:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23687168=0x1697000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.594608:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.594611:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23691264 00000020:00010000:1.0:1789760789.594619:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.594623:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.594636:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.594640:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.594643:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.594652:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.594659:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.594796:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760789.594904:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23687168<>23691264 00000100:00100000:0.0:1789760789.594986:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507377536:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760789.631878:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760789.631895:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.631908:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.631933:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760789.631967:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507377536:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760789.634406:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5783, 5783] 00000080:00200000:1.0:1789760789.634432:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.634440:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.634457:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23691264 i_blocks 47111 00000020:00200000:1.0:1789760789.634463:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.634479:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.634511:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23691264, count: 4096 00000080:00200000:1.0:1789760789.634520:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23691264, bytes: 4096 00000080:00200000:1.0:1789760789.634534:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.634544:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.634546:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23691264, 23695360) 0 0 00000080:00200000:1.0:1789760789.634550:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.634568:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5784, 5784] 00020000:00200000:1.0:1789760789.634577:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23691264 chunk: [23691264, 23695360) 23695360, 4096 00020000:00200000:1.0:1789760789.634582:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.634604:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5784, 5784] 00000008:00010000:1.0:1789760789.634623:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.634646:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.634658:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23691264, 23695360) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.634676:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23691264=0x1698000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.634742:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.634746:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23695360 00000020:00010000:1.0:1789760789.634754:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.634758:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.634762:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.634766:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.634769:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.634779:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.634786:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.634806:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760789.634912:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23691264<>23695360 00000100:00100000:0.0:1789760789.636823:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507377664:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.704900:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973104 gen 3 00000008:00010000:0.0:1789760789.704978:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760789.704988:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.704999:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.705025:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760789.705057:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507377664:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760789.705161:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5784, 5784] 00000080:00200000:1.0:1789760789.705178:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.705184:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.705200:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23695360 i_blocks 47111 00000020:00200000:1.0:1789760789.705204:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.705214:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.705249:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23695360, count: 4096 00000080:00200000:1.0:1789760789.705256:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23695360, bytes: 4096 00000080:00200000:1.0:1789760789.705266:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.705274:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.705276:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23695360, 23699456) 0 0 00000080:00200000:1.0:1789760789.705280:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.705297:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5785, 5785] 00020000:00200000:1.0:1789760789.705305:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23695360 chunk: [23695360, 23699456) 23699456, 4096 00020000:00200000:1.0:1789760789.705310:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.705328:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5785, 5785] 00000008:00010000:1.0:1789760789.705346:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.705352:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.705365:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23695360, 23699456) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.705413:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23695360=0x1699000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.705468:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.705471:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23699456 00000020:00010000:1.0:1789760789.705481:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.705484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.705487:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.705490:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.705493:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.705501:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.705510:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.705527:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760789.705647:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23695360<>23699456 00000100:00100000:0.0:1789760789.705886:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507377792:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760789.779117:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760789.779134:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.779147:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.779178:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760789.779209:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507377792:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760789.779337:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5785, 5785] 00000080:00200000:1.0:1789760789.779354:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.779361:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.779441:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23699456 i_blocks 47111 00000020:00200000:1.0:1789760789.779446:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.779480:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.779516:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23699456, count: 4096 00000080:00200000:1.0:1789760789.779524:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23699456, bytes: 4096 00000080:00200000:1.0:1789760789.779539:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.779548:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.779550:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23699456, 23703552) 0 0 00000080:00200000:1.0:1789760789.779555:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.779573:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5786, 5786] 00020000:00200000:1.0:1789760789.779582:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23699456 chunk: [23699456, 23703552) 23703552, 4096 00020000:00200000:1.0:1789760789.779587:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.779607:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5786, 5786] 00000008:00010000:1.0:1789760789.779640:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.779647:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.779660:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23699456, 23703552) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.779675:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23699456=0x169a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.779734:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.779737:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23703552 00000020:00010000:1.0:1789760789.779744:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.779748:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.779751:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.779755:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.779757:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.779766:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.779772:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.779786:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760789.779894:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23699456<>23703552 00000100:00100000:0.0:1789760789.780091:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507377920:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.829977:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973106 gen 3 00000008:00010000:0.0:1789760789.830040:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760789.830049:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.830058:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.830080:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760789.830108:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507377920:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760789.830256:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5786, 5786] 00000080:00200000:1.0:1789760789.830271:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.830290:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.830307:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23703552 i_blocks 47111 00000020:00200000:1.0:1789760789.830311:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.830322:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.830362:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23703552, count: 4096 00000080:00200000:1.0:1789760789.830369:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23703552, bytes: 4096 00000080:00200000:1.0:1789760789.830414:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.830425:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.830428:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23703552, 23707648) 0 0 00000080:00200000:1.0:1789760789.830432:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.830449:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5787, 5787] 00020000:00200000:1.0:1789760789.830458:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23703552 chunk: [23703552, 23707648) 23707648, 4096 00020000:00200000:1.0:1789760789.830462:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.830483:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5787, 5787] 00000008:00010000:1.0:1789760789.830502:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.830507:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.830791:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23703552, 23707648) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.830808:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23703552=0x169b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.830869:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.830872:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23707648 00000020:00010000:1.0:1789760789.830881:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.830884:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.830887:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.830891:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.830958:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.830970:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.830977:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.830997:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760789.831106:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23703552<>23707648 00000100:00100000:0.0:1789760789.831219:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378048:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760789.901842:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760789.901859:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.901872:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.902079:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760789.902117:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378048:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760789.902267:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5787, 5787] 00000080:00200000:1.0:1789760789.902285:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.902293:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.902312:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23707648 i_blocks 47111 00000020:00200000:1.0:1789760789.902316:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.902327:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.902367:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23707648, count: 4096 00000080:00200000:1.0:1789760789.902413:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23707648, bytes: 4096 00000080:00200000:1.0:1789760789.902429:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.902441:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.902444:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23707648, 23711744) 0 0 00000080:00200000:1.0:1789760789.902449:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.902608:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5788, 5788] 00020000:00200000:1.0:1789760789.902621:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23707648 chunk: [23707648, 23711744) 23711744, 4096 00020000:00200000:1.0:1789760789.902636:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.902659:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5788, 5788] 00000008:00010000:1.0:1789760789.902677:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.902685:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.902698:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23707648, 23711744) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.902712:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23707648=0x169c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.902775:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.902778:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23711744 00000020:00010000:1.0:1789760789.902786:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.902790:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.902793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.902797:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.902800:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.902808:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.902815:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.902834:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760789.903143:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168a80 - object 0x280000401:3 offset 23707648<>23711744 00000100:00100000:0.0:1789760789.903230:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507378176:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.946309:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973108 gen 3 00000008:00010000:0.0:1789760789.946411:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760789.946420:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.946430:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.946546:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5788, 5788] 00000080:00200000:1.0:1789760789.946564:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760789.946571:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760789.946587:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23711744 i_blocks 47111 00000020:00200000:1.0:1789760789.946590:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760789.946601:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760789.946649:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23711744, count: 4096 00000080:00200000:1.0:1789760789.946659:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23711744, bytes: 4096 00000080:00200000:1.0:1789760789.946670:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760789.946679:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760789.946682:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23711744, 23715840) 0 0 00000080:00200000:1.0:1789760789.946685:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760789.946703:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5789, 5789] 00020000:00200000:1.0:1789760789.946711:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23711744 chunk: [23711744, 23715840) 23715840, 4096 00020000:00200000:1.0:1789760789.946716:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760789.946736:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5789, 5789] 00000008:00010000:1.0:1789760789.946758:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760789.946764:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760789.946776:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23711744, 23715840) append 0 append lockpos 0 00000080:00200000:1.0:1789760789.946790:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23711744=0x169d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760789.946862:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760789.946865:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23715840 00000020:00010000:1.0:1789760789.946873:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760789.946877:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.946879:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760789.946883:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760789.946886:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760789.946894:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760789.946901:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760789.946926:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760789.947043:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168380 - object 0x280000401:3 offset 23711744<>23715840 00000100:00100000:1.0:1789760789.947163:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378304:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760789.947879:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760789.947900:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507378176:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760789.988420:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973109 gen 3 00000008:00010000:0.0:1789760789.988474:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760789.988482:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760789.988493:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760789.988546:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760789.988577:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378304:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760789.988888:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5789, 5789] 00000080:00200000:2.0:1789760789.988906:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760789.988913:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760789.988929:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23715840 i_blocks 47111 00000020:00200000:2.0:1789760789.988933:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760789.988943:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760789.988974:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23715840, count: 4096 00000080:00200000:2.0:1789760789.988982:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23715840, bytes: 4096 00000080:00200000:2.0:1789760789.988993:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760789.989004:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760789.989007:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23715840, 23719936) 0 0 00000080:00200000:2.0:1789760789.989012:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760789.989032:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5790, 5790] 00020000:00200000:2.0:1789760789.989042:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23715840 chunk: [23715840, 23719936) 23719936, 4096 00020000:00200000:2.0:1789760789.989047:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760789.989129:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5790, 5790] 00000008:00010000:2.0:1789760789.989152:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760789.989158:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760789.989170:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23715840, 23719936) append 0 append lockpos 0 00000080:00200000:2.0:1789760789.989183:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23715840=0x169e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760789.989239:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760789.989241:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23719936 00000020:00010000:2.0:1789760789.989249:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760789.989253:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.989256:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760789.989259:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760789.989262:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760789.989269:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760789.989276:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760789.989294:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760789.989420:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 23715840<>23719936 00000100:00100000:2.0:1789760790.020111:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973110 gen 3 00000008:00010000:2.0:1789760790.020162:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac300 00000020:00200000:2.0:1789760790.020171:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.020182:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.020203:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac300 00000100:00100000:2.0:1789760790.020233:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507378432:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760790.020336:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5790, 5790] 00000080:00200000:1.0:1789760790.020354:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760790.020360:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760790.020405:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23719936 i_blocks 47111 00000020:00200000:1.0:1789760790.020410:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760790.020420:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760790.020455:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23719936, count: 4096 00000080:00200000:1.0:1789760790.020462:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23719936, bytes: 4096 00000080:00200000:1.0:1789760790.020472:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760790.020481:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760790.020484:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23719936, 23724032) 0 0 00000080:00200000:1.0:1789760790.020488:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760790.020505:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5791, 5791] 00020000:00200000:1.0:1789760790.020514:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23719936 chunk: [23719936, 23724032) 23724032, 4096 00020000:00200000:1.0:1789760790.020519:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760790.020538:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5791, 5791] 00000008:00010000:1.0:1789760790.020555:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760790.020560:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760790.020572:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23719936, 23724032) append 0 append lockpos 0 00000080:00200000:1.0:1789760790.020588:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23719936=0x169f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760790.020655:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760790.020667:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23724032 00000020:00010000:1.0:1789760790.020675:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760790.020679:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.020682:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.020686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760790.020689:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760790.020698:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760790.020705:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760790.020722:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760790.020828:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a300 - object 0x280000401:3 offset 23719936<>23724032 00000100:00100000:0.0:1789760790.021044:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760790.021065:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378560:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760790.058265:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760790.058282:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.058294:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.058316:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760790.058347:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378560:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760790.058517:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5791, 5791] 00000080:00200000:1.0:1789760790.058533:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760790.058540:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760790.058556:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23724032 i_blocks 47111 00000020:00200000:1.0:1789760790.058560:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760790.058569:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760790.058604:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23724032, count: 4096 00000080:00200000:1.0:1789760790.058611:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23724032, bytes: 4096 00000080:00200000:1.0:1789760790.058631:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760790.058641:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760790.058644:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23724032, 23728128) 0 0 00000080:00200000:1.0:1789760790.058648:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760790.058729:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5792, 5792] 00020000:00200000:1.0:1789760790.058738:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23724032 chunk: [23724032, 23728128) 23728128, 4096 00020000:00200000:1.0:1789760790.058743:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760790.058762:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5792, 5792] 00000008:00010000:1.0:1789760790.058781:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760790.058787:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760790.058798:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23724032, 23728128) append 0 append lockpos 0 00000080:00200000:1.0:1789760790.058812:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23724032=0x16a0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760790.058870:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760790.058874:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23728128 00000020:00010000:1.0:1789760790.058882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760790.058886:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.058889:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.058892:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760790.058939:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760790.058949:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760790.058956:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760790.058974:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760790.059138:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169c00 - object 0x280000401:3 offset 23724032<>23728128 00000100:00100000:0.0:1789760790.059202:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378688:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760790.090934:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973112 gen 3 00000008:00010000:0.0:1789760790.091001:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760790.091010:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.091020:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.091041:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760790.091139:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507378688:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760790.091238:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5792, 5792] 00000080:00200000:1.0:1789760790.091254:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760790.091260:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760790.091276:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23728128 i_blocks 47111 00000020:00200000:1.0:1789760790.091280:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760790.091290:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760790.091323:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23728128, count: 4096 00000080:00200000:1.0:1789760790.091329:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23728128, bytes: 4096 00000080:00200000:1.0:1789760790.091340:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760790.091349:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760790.091351:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23728128, 23732224) 0 0 00000080:00200000:1.0:1789760790.091355:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760790.091372:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5793, 5793] 00020000:00200000:1.0:1789760790.091409:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23728128 chunk: [23728128, 23732224) 23732224, 4096 00020000:00200000:1.0:1789760790.091414:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760790.091433:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5793, 5793] 00000008:00010000:1.0:1789760790.091452:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760790.091458:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760790.091471:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23728128, 23732224) append 0 append lockpos 0 00000080:00200000:1.0:1789760790.091486:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23728128=0x16a1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760790.091543:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760790.091546:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23732224 00000020:00010000:1.0:1789760790.091554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760790.091558:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.091561:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.091565:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760790.091568:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760790.091577:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760790.091583:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760790.091599:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760790.091780:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 23728128<>23732224 00000100:00100000:0.0:1789760790.092020:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507378816:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760790.119984:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973113 gen 3 00000008:00010000:0.0:1789760790.120037:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760790.120046:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.120057:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.120136:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760790.120166:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507378816:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760790.120272:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5793, 5793] 00000080:00200000:1.0:1789760790.120289:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760790.120295:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760790.120311:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23732224 i_blocks 47111 00000020:00200000:1.0:1789760790.120315:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760790.120325:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760790.120359:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23732224, count: 4096 00000080:00200000:1.0:1789760790.120366:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23732224, bytes: 4096 00000080:00200000:1.0:1789760790.120409:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760790.120419:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760790.120421:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23732224, 23736320) 0 0 00000080:00200000:1.0:1789760790.120426:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760790.120441:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5794, 5794] 00020000:00200000:1.0:1789760790.120449:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23732224 chunk: [23732224, 23736320) 23736320, 4096 00020000:00200000:1.0:1789760790.120454:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760790.120474:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5794, 5794] 00000008:00010000:1.0:1789760790.120491:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760790.120497:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760790.120509:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23732224, 23736320) append 0 append lockpos 0 00000080:00200000:1.0:1789760790.120522:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23732224=0x16a2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760790.120576:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760790.120579:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23736320 00000020:00010000:1.0:1789760790.120588:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760790.120591:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.120595:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.120599:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760790.120602:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760790.120610:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760790.120618:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760790.120647:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760790.120822:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 23732224<>23736320 00000100:00100000:0.0:1789760790.120975:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760790.120993:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507378944:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760790.161888:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973114 gen 3 00000008:00010000:0.0:1789760790.161942:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760790.161951:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.161974:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.161994:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760790.162023:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507378944:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760790.162452:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5794, 5794] 00000080:00200000:1.0:1789760790.162472:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760790.162479:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760790.162498:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23736320 i_blocks 47111 00000020:00200000:1.0:1789760790.162503:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760790.162513:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760790.162549:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23736320, count: 4096 00000080:00200000:1.0:1789760790.162558:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23736320, bytes: 4096 00000080:00200000:1.0:1789760790.162569:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760790.162578:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760790.162581:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23736320, 23740416) 0 0 00000080:00200000:1.0:1789760790.162585:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760790.162602:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5795, 5795] 00020000:00200000:1.0:1789760790.162612:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23736320 chunk: [23736320, 23740416) 23740416, 4096 00020000:00200000:1.0:1789760790.162617:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760790.162651:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5795, 5795] 00000008:00010000:1.0:1789760790.162673:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760790.162679:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760790.162694:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23736320, 23740416) append 0 append lockpos 0 00000080:00200000:1.0:1789760790.162711:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23736320=0x16a3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760790.162776:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760790.162780:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23740416 00000020:00010000:1.0:1789760790.162789:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760790.162793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.162796:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.162800:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760790.162803:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760790.162811:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760790.162818:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760790.162840:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760790.162967:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23736320<>23740416 00000100:00100000:0.0:1789760790.163322:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507379072:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760790.197252:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760790.197266:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.197277:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.197298:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760790.197327:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507379072:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760790.197413:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5795, 5795] 00000080:00200000:1.0:1789760790.197430:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760790.197436:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760790.197454:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23740416 i_blocks 47111 00000020:00200000:1.0:1789760790.197457:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760790.197467:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760790.197503:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23740416, count: 4096 00000080:00200000:1.0:1789760790.197511:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23740416, bytes: 4096 00000080:00200000:1.0:1789760790.197520:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760790.197530:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760790.197533:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23740416, 23744512) 0 0 00000080:00200000:1.0:1789760790.197537:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760790.197554:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5796, 5796] 00020000:00200000:1.0:1789760790.197563:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23740416 chunk: [23740416, 23744512) 23744512, 4096 00020000:00200000:1.0:1789760790.197568:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760790.197586:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5796, 5796] 00000008:00010000:1.0:1789760790.197601:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760790.197606:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760790.197617:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23740416, 23744512) append 0 append lockpos 0 00000080:00200000:1.0:1789760790.197639:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23740416=0x16a4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760790.197698:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760790.197701:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23744512 00000020:00010000:1.0:1789760790.197708:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760790.197712:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.197715:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.197719:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760790.197722:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760790.197728:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760790.197734:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760790.197749:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760790.197839:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23740416<>23744512 00000100:00100000:0.0:1789760790.198035:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507379200:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760790.255495:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973116 gen 3 00000008:00010000:0.0:1789760790.255603:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760790.255613:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.255632:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.255704:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760790.255737:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507379200:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760790.256544:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5796, 5796] 00000080:00200000:1.0:1789760790.256565:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760790.256572:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760790.256588:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23744512 i_blocks 47111 00000020:00200000:1.0:1789760790.256592:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760790.256604:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760790.256648:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23744512, count: 4096 00000080:00200000:1.0:1789760790.256656:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23744512, bytes: 4096 00000080:00200000:1.0:1789760790.257079:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760790.257090:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760790.257092:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23744512, 23748608) 0 0 00000080:00200000:1.0:1789760790.257097:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760790.257114:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5797, 5797] 00020000:00200000:1.0:1789760790.257122:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23744512 chunk: [23744512, 23748608) 23748608, 4096 00020000:00200000:1.0:1789760790.257126:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760790.257157:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5797, 5797] 00000008:00010000:1.0:1789760790.257186:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760790.257192:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760790.257204:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23744512, 23748608) append 0 append lockpos 0 00000080:00200000:1.0:1789760790.257226:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23744512=0x16a5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760790.257293:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760790.257296:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23748608 00000020:00010000:1.0:1789760790.257304:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:1.0:1789760790.257308:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.257311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760790.257314:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760790.257317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760790.257326:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760790.257333:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760790.257361:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760790.257876:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23744512<>23748608 00000100:00100000:0.0:1789760790.257949:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507379328:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760790.292433:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973117 gen 3 00000008:00010000:0.0:1789760790.292488:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760790.292497:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.292509:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.292542:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760790.292572:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507379328:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760790.292760:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5797, 5797] 00000080:00200000:2.0:1789760790.292777:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760790.292784:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760790.292800:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23748608 i_blocks 47111 00000020:00200000:2.0:1789760790.292804:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760790.292814:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760790.292847:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23748608, count: 4096 00000080:00200000:2.0:1789760790.292854:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23748608, bytes: 4096 00000080:00200000:2.0:1789760790.292868:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760790.292877:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760790.292880:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23748608, 23752704) 0 0 00000080:00200000:2.0:1789760790.292884:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760790.292902:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5798, 5798] 00020000:00200000:2.0:1789760790.292911:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23748608 chunk: [23748608, 23752704) 23752704, 4096 00020000:00200000:2.0:1789760790.292915:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760790.292936:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5798, 5798] 00000008:00010000:2.0:1789760790.292955:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760790.292961:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760790.292972:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23748608, 23752704) append 0 append lockpos 0 00000080:00200000:2.0:1789760790.292988:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23748608=0x16a6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760790.293047:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760790.293050:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23752704 00000020:00010000:2.0:1789760790.293058:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760790.293062:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.293065:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.293068:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760790.293070:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760790.293078:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760790.293085:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760790.293102:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760790.293210:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 23748608<>23752704 00000100:00100000:2.0:1789760790.293253:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760790.293270:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507379456:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760790.349829:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac300 00000020:00200000:2.0:1789760790.349845:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.349858:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.349887:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac300 00000080:00200000:0.0:1789760790.349997:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5798, 5798] 00000080:00200000:0.0:1789760790.350016:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760790.350022:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760790.350038:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23752704 i_blocks 47111 00000020:00200000:0.0:1789760790.350042:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760790.350052:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760790.350088:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23752704, count: 4096 00000080:00200000:0.0:1789760790.350096:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23752704, bytes: 4096 00000080:00200000:0.0:1789760790.350106:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760790.350115:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760790.350118:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23752704, 23756800) 0 0 00000080:00200000:0.0:1789760790.350123:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000100:00100000:2.0:1789760790.350271:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507379456:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760790.350361:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5799, 5799] 00020000:00200000:0.0:1789760790.350370:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23752704 chunk: [23752704, 23756800) 23756800, 4096 00020000:00200000:0.0:1789760790.350375:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760790.350439:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5799, 5799] 00000008:00010000:0.0:1789760790.350460:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760790.350466:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760790.350479:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23752704, 23756800) append 0 append lockpos 0 00000080:00200000:0.0:1789760790.350493:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23752704=0x16a7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760790.350558:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760790.350562:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23756800 00000020:00010000:0.0:1789760790.350571:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:0.0:1789760790.350575:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760790.350578:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760790.350581:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760790.350584:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760790.350603:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760790.350610:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760790.350634:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760790.350740:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23752704<>23756800 00000100:00100000:0.0:1789760790.350784:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760790.350799:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507379584:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760790.393734:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973119 gen 3 00000008:00010000:0.0:1789760790.393799:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760790.393809:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760790.393821:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760790.394774:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760790.394811:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507379584:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760790.395053:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5799, 5799] 00000080:00200000:2.0:1789760790.395071:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760790.395077:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760790.395094:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23756800 i_blocks 47111 00000020:00200000:2.0:1789760790.395098:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760790.395109:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760790.395139:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23756800, count: 4096 00000080:00200000:2.0:1789760790.395147:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23756800, bytes: 4096 00000080:00200000:2.0:1789760790.395160:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760790.395168:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760790.395170:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23756800, 23760896) 0 0 00000080:00200000:2.0:1789760790.395175:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760790.395192:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5800, 5800] 00020000:00200000:2.0:1789760790.395201:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23756800 chunk: [23756800, 23760896) 23760896, 4096 00020000:00200000:2.0:1789760790.395205:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760790.395224:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5800, 5800] 00000008:00010000:2.0:1789760790.395242:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760790.395247:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760790.395259:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23756800, 23760896) append 0 append lockpos 0 00000080:00200000:2.0:1789760790.395337:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23756800=0x16a8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760790.395416:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760790.395420:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23760896 00000020:00010000:2.0:1789760790.395430:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760790.395433:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.395436:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.395502:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760790.395506:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760790.395515:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760790.395523:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760790.395540:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760790.395728:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 23756800<>23760896 00000080:00200000:2.0:1789760790.443599:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5800, 5800] 00000080:00200000:2.0:1789760790.443614:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760790.443627:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760790.443643:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23760896 i_blocks 47111 00000020:00200000:2.0:1789760790.443646:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760790.443663:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760790.443798:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23760896, count: 4096 00000080:00200000:2.0:1789760790.443807:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23760896, bytes: 4096 00000080:00200000:2.0:1789760790.443818:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760790.443827:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760790.443829:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23760896, 23764992) 0 0 00000080:00200000:2.0:1789760790.443833:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760790.443848:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5801, 5801] 00020000:00200000:2.0:1789760790.443855:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23760896 chunk: [23760896, 23764992) 23764992, 4096 00020000:00200000:2.0:1789760790.443858:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760790.443876:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5801, 5801] 00000008:00010000:2.0:1789760790.443893:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760790.443899:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760790.443911:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23760896, 23764992) append 0 append lockpos 0 00000080:00200000:2.0:1789760790.443926:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23760896=0x16a9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760790.443986:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760790.443989:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23764992 00000020:00010000:2.0:1789760790.443997:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760790.444001:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.444004:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.444008:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760790.444011:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760790.444019:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760790.444026:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760790.444044:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760790.444228:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 23760896<>23764992 00000080:00200000:2.0:1789760790.490444:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5801, 5801] 00000080:00200000:2.0:1789760790.490460:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760790.490465:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760790.490477:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23764992 i_blocks 47111 00000020:00200000:2.0:1789760790.490480:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760790.490488:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760790.490912:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23764992, count: 4096 00000080:00200000:2.0:1789760790.490920:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23764992, bytes: 4096 00000080:00200000:2.0:1789760790.490935:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760790.490946:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760790.490948:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23764992, 23769088) 0 0 00000080:00200000:2.0:1789760790.490953:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760790.490973:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5802, 5802] 00020000:00200000:2.0:1789760790.490982:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23764992 chunk: [23764992, 23769088) 23769088, 4096 00020000:00200000:2.0:1789760790.490986:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760790.491008:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5802, 5802] 00000008:00010000:2.0:1789760790.491026:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760790.491033:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760790.491044:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23764992, 23769088) append 0 append lockpos 0 00000080:00200000:2.0:1789760790.491058:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23764992=0x16aa000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760790.491129:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760790.491132:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23769088 00000020:00010000:2.0:1789760790.491141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760790.491145:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.491148:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760790.491152:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038bd8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760790.491155:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760790.491163:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760790.491170:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760790.491188:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760790.491295:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 23764992<>23769088 00000100:00100000:2.0:1789760790.491338:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760790.491811:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507379968:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.543636:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973122 gen 3 00000008:00010000:2.0:1789760790.543713:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50acc00 00000020:00200000:2.0:1789760790.543723:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.543733:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.543755:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50acc00 00000100:00100000:2.0:1789760790.543787:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507379968:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.544519:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380096:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.579642:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973123 gen 3 00000008:00010000:2.0:1789760790.579788:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c240 00000020:00200000:2.0:1789760790.579797:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.579808:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.579832:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c240 00000100:00100000:2.0:1789760790.579862:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380096:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.580541:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380224:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.623353:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973124 gen 3 00000008:00010000:2.0:1789760790.623443:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000020:00200000:2.0:1789760790.623452:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.623463:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.623485:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000100:00100000:2.0:1789760790.623514:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380224:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.624284:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760790.624301:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380352:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.668675:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973125 gen 3 00000008:00010000:2.0:1789760790.668729:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5ce40 00000020:00200000:2.0:1789760790.668737:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.668748:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.668775:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5ce40 00000100:00100000:2.0:1789760790.668810:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380352:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.669648:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380480:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760790.722981:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5ce40 00000020:00200000:2.0:1789760790.722996:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.723007:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.723029:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5ce40 00000100:00100000:2.0:1789760790.723058:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380480:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.723743:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380608:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.764256:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973126 gen 3 00000100:00100000:2.0:1789760790.764304:0:2418:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d1b08380 x1876699507380608/t4294973127(4294973127) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760806 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:2.0:1789760790.764352:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000020:00200000:2.0:1789760790.764360:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.764369:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.764422:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000100:00100000:2.0:1789760790.765354:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380608:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.766296:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760790.766316:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380736:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.815542:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973128 gen 3 00000008:00010000:2.0:1789760790.815611:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c240 00000020:00200000:2.0:1789760790.815632:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.815643:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.815664:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c240 00000100:00100000:2.0:1789760790.815694:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380736:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.816605:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380864:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.862649:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973129 gen 3 00000008:00010000:2.0:1789760790.862701:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c3c0 00000020:00200000:2.0:1789760790.862709:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.862735:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.862754:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c3c0 00000100:00100000:2.0:1789760790.862782:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507380864:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.863976:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380992:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.917940:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973130 gen 3 00000008:00010000:2.0:1789760790.918035:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5cf00 00000020:00200000:2.0:1789760790.918044:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.918054:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.918295:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5cf00 00000100:00100000:2.0:1789760790.918330:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507380992:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.919288:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507381120:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.967851:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973131 gen 3 00000008:00010000:2.0:1789760790.967956:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c180 00000020:00200000:2.0:1789760790.967965:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760790.968037:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760790.968160:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c180 00000100:00100000:2.0:1789760790.968193:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507381120:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760790.969286:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507381248:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760791.008335:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c000 00000020:00200000:2.0:1789760791.008352:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760791.008366:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760791.008414:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c000 00000100:00100000:2.0:1789760791.008446:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507381248:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760791.009265:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507381376:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760791.051354:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973133 gen 3 00000008:00010000:2.0:1789760791.051454:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c300 00000020:00200000:2.0:1789760791.051463:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760791.051473:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760791.051495:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c300 00000100:00100000:2.0:1789760791.051524:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507381376:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760791.052368:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760791.052419:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507381504:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760791.086971:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973134 gen 3 00000008:00010000:2.0:1789760791.087025:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5cd80 00000020:00200000:2.0:1789760791.087034:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760791.087045:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760791.087169:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5cd80 00000100:00100000:2.0:1789760791.087203:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507381504:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760791.087414:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23818240 i_blocks 47111 00000020:00200000:2.0:1789760791.087419:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760791.087430:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760791.087467:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23818240, count: 4096 00000080:00200000:2.0:1789760791.087475:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23818240, bytes: 4096 00000080:00200000:2.0:1789760791.087486:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760791.087496:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760791.087499:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23818240, 23822336) 0 0 00000080:00200000:2.0:1789760791.087504:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760791.087522:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5815, 5815] 00020000:00200000:2.0:1789760791.087531:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23818240 chunk: [23818240, 23822336) 23822336, 4096 00020000:00200000:2.0:1789760791.087536:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760791.087554:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5815, 5815] 00000008:00010000:2.0:1789760791.087571:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760791.087576:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760791.087588:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23818240, 23822336) append 0 append lockpos 0 00000080:00200000:2.0:1789760791.087603:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23818240=0x16b7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760791.087782:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760791.087786:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23822336 00000020:00010000:2.0:1789760791.087796:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760791.087799:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.087802:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.087805:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760791.087808:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760791.087816:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760791.087823:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760791.087839:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760791.087964:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 23818240<>23822336 00000100:00100000:2.0:1789760791.088012:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760791.088034:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507381632:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760791.124348:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973135 gen 3 00000008:00010000:2.0:1789760791.124439:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50acc00 00000020:00200000:2.0:1789760791.124448:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760791.124459:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760791.124488:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50acc00 00000100:00100000:2.0:1789760791.124785:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507381632:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760791.126007:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5815, 5815] 00000080:00200000:0.0:1789760791.126031:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760791.126038:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760791.126057:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23822336 i_blocks 47111 00000020:00200000:0.0:1789760791.126063:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760791.126077:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760791.126113:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23822336, count: 4096 00000080:00200000:0.0:1789760791.126121:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23822336, bytes: 4096 00000080:00200000:0.0:1789760791.126135:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760791.126145:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760791.126148:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23822336, 23826432) 0 0 00000080:00200000:0.0:1789760791.126152:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760791.126169:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5816, 5816] 00020000:00200000:0.0:1789760791.126178:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23822336 chunk: [23822336, 23826432) 23826432, 4096 00020000:00200000:0.0:1789760791.126181:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760791.126203:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5816, 5816] 00000008:00010000:0.0:1789760791.126221:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760791.126227:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760791.126239:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23822336, 23826432) append 0 append lockpos 0 00000080:00200000:0.0:1789760791.126253:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23822336=0x16b8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760791.126321:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760791.126324:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23826432 00000020:00010000:0.0:1789760791.126331:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760791.126335:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760791.126793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760791.126800:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760791.126803:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760791.126817:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760791.126825:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760791.127759:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760791.127976:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23822336<>23826432 00000100:00100000:0.0:1789760791.128025:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760791.128051:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507381760:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.167335:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973136 gen 3 00000008:00010000:0.0:1789760791.167423:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760791.167432:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.167444:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.167469:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760791.167499:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507381760:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760791.167882:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5816, 5816] 00000080:00200000:2.0:1789760791.167912:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760791.167919:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760791.167938:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23826432 i_blocks 47111 00000020:00200000:2.0:1789760791.167941:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760791.167953:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760791.167990:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23826432, count: 4096 00000080:00200000:2.0:1789760791.167998:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23826432, bytes: 4096 00000080:00200000:2.0:1789760791.168009:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760791.168018:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760791.168021:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23826432, 23830528) 0 0 00000080:00200000:2.0:1789760791.168025:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760791.168042:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5817, 5817] 00020000:00200000:2.0:1789760791.168051:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23826432 chunk: [23826432, 23830528) 23830528, 4096 00020000:00200000:2.0:1789760791.168056:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760791.168178:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5817, 5817] 00000008:00010000:2.0:1789760791.168197:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760791.168204:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760791.168217:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23826432, 23830528) append 0 append lockpos 0 00000080:00200000:2.0:1789760791.168231:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23826432=0x16b9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760791.168293:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760791.168296:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23830528 00000020:00010000:2.0:1789760791.168305:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760791.168309:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.168312:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.168317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760791.168320:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760791.168328:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760791.168336:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760791.168355:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760791.168486:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 23826432<>23830528 00000080:00200000:2.0:1789760791.212830:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5817, 5817] 00000080:00200000:2.0:1789760791.212848:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760791.212855:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760791.212871:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23830528 i_blocks 47111 00000020:00200000:2.0:1789760791.212876:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760791.212888:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760791.212923:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23830528, count: 4096 00000080:00200000:2.0:1789760791.212931:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23830528, bytes: 4096 00000080:00200000:2.0:1789760791.212944:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760791.212955:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760791.212958:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23830528, 23834624) 0 0 00000080:00200000:2.0:1789760791.212963:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760791.212981:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5818, 5818] 00020000:00200000:2.0:1789760791.212991:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23830528 chunk: [23830528, 23834624) 23834624, 4096 00020000:00200000:2.0:1789760791.212996:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760791.213017:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5818, 5818] 00000008:00010000:2.0:1789760791.213035:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760791.213042:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760791.213055:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23830528, 23834624) append 0 append lockpos 0 00000080:00200000:2.0:1789760791.213072:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23830528=0x16ba000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760791.213347:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760791.213351:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23834624 00000020:00010000:2.0:1789760791.213361:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760791.213364:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.213368:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.213371:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760791.213374:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760791.213795:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760791.213804:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760791.213827:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760791.213932:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 23830528<>23834624 00000080:00200000:2.0:1789760791.292257:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5818, 5818] 00000080:00200000:2.0:1789760791.292275:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760791.292282:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760791.292299:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23834624 i_blocks 47111 00000020:00200000:2.0:1789760791.292304:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760791.292315:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760791.292351:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23834624, count: 4096 00000080:00200000:2.0:1789760791.292359:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23834624, bytes: 4096 00000080:00200000:2.0:1789760791.292370:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760791.292410:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760791.292413:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23834624, 23838720) 0 0 00000080:00200000:2.0:1789760791.292418:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760791.292434:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5819, 5819] 00020000:00200000:2.0:1789760791.292443:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23834624 chunk: [23834624, 23838720) 23838720, 4096 00020000:00200000:2.0:1789760791.292447:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760791.292469:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5819, 5819] 00000008:00010000:2.0:1789760791.292488:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760791.292493:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760791.292505:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23834624, 23838720) append 0 append lockpos 0 00000080:00200000:2.0:1789760791.292520:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23834624=0x16bb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760791.292578:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760791.292582:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23838720 00000020:00010000:2.0:1789760791.292591:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760791.292594:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.292597:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.292601:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760791.292603:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760791.292621:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760791.292629:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760791.292645:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760791.293232:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 23834624<>23838720 00000080:00200000:2.0:1789760791.330264:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5819, 5819] 00000080:00200000:2.0:1789760791.330281:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760791.330288:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760791.330304:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23838720 i_blocks 47111 00000020:00200000:2.0:1789760791.330307:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760791.330317:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760791.330352:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23838720, count: 4096 00000080:00200000:2.0:1789760791.330360:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23838720, bytes: 4096 00000080:00200000:2.0:1789760791.330369:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760791.330414:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760791.330417:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23838720, 23842816) 0 0 00000080:00200000:2.0:1789760791.330422:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760791.330438:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5820, 5820] 00020000:00200000:2.0:1789760791.330446:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23838720 chunk: [23838720, 23842816) 23842816, 4096 00020000:00200000:2.0:1789760791.330451:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760791.330473:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5820, 5820] 00000008:00010000:2.0:1789760791.330491:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760791.330497:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760791.330508:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23838720, 23842816) append 0 append lockpos 0 00000080:00200000:2.0:1789760791.330522:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23838720=0x16bc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760791.330579:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760791.330582:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23842816 00000020:00010000:2.0:1789760791.330590:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760791.330593:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.330596:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.330600:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760791.330603:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760791.330610:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760791.330752:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760791.330771:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760791.330876:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 23838720<>23842816 00000080:00200000:0.0:1789760791.362338:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5820, 5820] 00000080:00200000:0.0:1789760791.362359:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760791.362364:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760791.362404:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23842816 i_blocks 47111 00000020:00200000:0.0:1789760791.362407:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760791.362416:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760791.362446:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23842816, count: 4096 00000080:00200000:0.0:1789760791.362451:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23842816, bytes: 4096 00000080:00200000:0.0:1789760791.362459:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760791.362466:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760791.362468:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23842816, 23846912) 0 0 00000080:00200000:0.0:1789760791.362472:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760791.362487:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5821, 5821] 00020000:00200000:0.0:1789760791.362494:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23842816 chunk: [23842816, 23846912) 23846912, 4096 00020000:00200000:0.0:1789760791.362497:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760791.362513:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5821, 5821] 00000008:00010000:0.0:1789760791.362527:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760791.362533:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760791.362544:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23842816, 23846912) append 0 append lockpos 0 00000080:00200000:0.0:1789760791.362560:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23842816=0x16bd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760791.362626:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760791.362629:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23846912 00000020:00010000:0.0:1789760791.362637:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760791.362641:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760791.362644:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760791.362648:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760791.362650:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760791.362658:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760791.362665:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760791.362689:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760791.362792:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23842816<>23846912 00000100:00100000:0.0:1789760791.362841:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760791.362861:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507382400:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.397836:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973141 gen 3 00000008:00010000:0.0:1789760791.397886:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760791.397895:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.397907:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.397930:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760791.397968:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507382400:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.399428:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5821, 5821] 00000080:00200000:1.0:1789760791.399452:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.399460:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.399477:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23846912 i_blocks 47111 00000020:00200000:1.0:1789760791.399482:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.399496:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.399529:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23846912, count: 4096 00000080:00200000:1.0:1789760791.399537:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23846912, bytes: 4096 00000080:00200000:1.0:1789760791.399550:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.399560:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.399562:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23846912, 23851008) 0 0 00000080:00200000:1.0:1789760791.399566:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.399582:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5822, 5822] 00020000:00200000:1.0:1789760791.399592:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23846912 chunk: [23846912, 23851008) 23851008, 4096 00020000:00200000:1.0:1789760791.399596:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.399629:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5822, 5822] 00000008:00010000:1.0:1789760791.399660:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.399667:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.399680:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23846912, 23851008) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.399695:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23846912=0x16be000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.399774:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.399777:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23851008 00000020:00010000:1.0:1789760791.399786:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.399790:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.399793:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.399796:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.399799:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.399808:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.399815:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.399835:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760791.399938:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23846912<>23851008 00000100:00100000:1.0:1789760791.399992:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760791.400017:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507382528:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.466128:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973142 gen 3 00000008:00010000:0.0:1789760791.466183:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760791.466192:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.466203:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.466225:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760791.466259:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507382528:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.466430:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5822, 5822] 00000080:00200000:1.0:1789760791.466449:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.466455:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.466472:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23851008 i_blocks 47111 00000020:00200000:1.0:1789760791.466477:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.466489:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.466529:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23851008, count: 4096 00000080:00200000:1.0:1789760791.466538:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23851008, bytes: 4096 00000080:00200000:1.0:1789760791.466549:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.466559:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.466561:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23851008, 23855104) 0 0 00000080:00200000:1.0:1789760791.466566:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.466583:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5823, 5823] 00020000:00200000:1.0:1789760791.466592:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23851008 chunk: [23851008, 23855104) 23855104, 4096 00020000:00200000:1.0:1789760791.466598:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.469137:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5823, 5823] 00000008:00010000:1.0:1789760791.469177:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.469184:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.469196:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23851008, 23855104) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.469214:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23851008=0x16bf000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.469293:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.469297:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23855104 00000020:00010000:1.0:1789760791.469307:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.469310:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.469313:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.469317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.469320:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.469333:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.469343:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.469413:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760791.469528:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23851008<>23855104 00000100:00100000:0.0:1789760791.471020:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507382656:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.501430:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973143 gen 3 00000008:00010000:0.0:1789760791.501489:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760791.501498:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.501510:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.501530:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760791.501561:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507382656:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.501710:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5823, 5823] 00000080:00200000:1.0:1789760791.501728:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.501734:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.501751:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23855104 i_blocks 47111 00000020:00200000:1.0:1789760791.501755:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.501766:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.501803:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23855104, count: 4096 00000080:00200000:1.0:1789760791.501810:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23855104, bytes: 4096 00000080:00200000:1.0:1789760791.501820:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.501829:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.501831:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23855104, 23859200) 0 0 00000080:00200000:1.0:1789760791.501835:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.501854:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5824, 5824] 00020000:00200000:1.0:1789760791.501862:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23855104 chunk: [23855104, 23859200) 23859200, 4096 00020000:00200000:1.0:1789760791.501866:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.501885:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5824, 5824] 00000008:00010000:1.0:1789760791.501943:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.501949:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.501959:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23855104, 23859200) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.501973:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23855104=0x16c0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.502026:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.502028:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23859200 00000020:00010000:1.0:1789760791.502038:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.502041:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.502045:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.502048:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.502051:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.502134:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.502140:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.502154:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760791.502250:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23855104<>23859200 00000100:00100000:0.0:1789760791.502312:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507382784:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760791.536094:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760791.536112:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.536123:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.536144:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760791.536174:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507382784:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.536248:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5824, 5824] 00000080:00200000:1.0:1789760791.536262:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.536267:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.536280:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23859200 i_blocks 47111 00000020:00200000:1.0:1789760791.536283:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.536292:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.536322:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23859200, count: 4096 00000080:00200000:1.0:1789760791.536328:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23859200, bytes: 4096 00000080:00200000:1.0:1789760791.536335:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.536342:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.536344:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23859200, 23863296) 0 0 00000080:00200000:1.0:1789760791.536348:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.536362:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5825, 5825] 00020000:00200000:1.0:1789760791.536370:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23859200 chunk: [23859200, 23863296) 23863296, 4096 00020000:00200000:1.0:1789760791.536373:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.536430:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5825, 5825] 00000008:00010000:1.0:1789760791.536446:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.536451:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.536462:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23859200, 23863296) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.536475:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23859200=0x16c1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.536532:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.536535:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23863296 00000020:00010000:1.0:1789760791.536543:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.536546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.536549:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.536553:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.536555:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.536564:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.536571:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.536589:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760791.536704:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23859200<>23863296 00000100:00100000:0.0:1789760791.536795:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507382912:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.565920:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973144 gen 3 00000100:00100000:0.0:1789760791.565979:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8168e00 x1876699507382912/t4294973145(4294973145) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760807 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760791.566030:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760791.566037:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.566048:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.566071:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760791.566097:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507382912:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.566258:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5825, 5825] 00000080:00200000:1.0:1789760791.566275:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.566281:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.566295:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23863296 i_blocks 47111 00000020:00200000:1.0:1789760791.566299:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.566310:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.566345:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23863296, count: 4096 00000080:00200000:1.0:1789760791.566352:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23863296, bytes: 4096 00000080:00200000:1.0:1789760791.566363:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.566370:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.566373:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23863296, 23867392) 0 0 00000080:00200000:1.0:1789760791.566407:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.566423:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5826, 5826] 00020000:00200000:1.0:1789760791.566431:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23863296 chunk: [23863296, 23867392) 23867392, 4096 00020000:00200000:1.0:1789760791.566436:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.566457:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5826, 5826] 00000008:00010000:1.0:1789760791.566475:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.566482:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.566494:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23863296, 23867392) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.566509:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23863296=0x16c2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.566563:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.566566:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23867392 00000020:00010000:1.0:1789760791.566574:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.566577:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.566580:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.566583:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.566586:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.566594:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.566600:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.566628:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760791.566730:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 23863296<>23867392 00000100:00100000:0.0:1789760791.567092:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760791.567112:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383040:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.603239:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973145 gen 3 00000100:00100000:0.0:1789760791.603653:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8168700 x1876699507383040/t4294973146(4294973146) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760807 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760791.604111:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760791.604117:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.604136:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.604159:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760791.604191:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383040:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.604410:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5826, 5826] 00000080:00200000:1.0:1789760791.604427:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.604432:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.604448:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23867392 i_blocks 47111 00000020:00200000:1.0:1789760791.604452:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.604462:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.604499:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23867392, count: 4096 00000080:00200000:1.0:1789760791.604507:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23867392, bytes: 4096 00000080:00200000:1.0:1789760791.604521:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.604542:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.604545:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23867392, 23871488) 0 0 00000080:00200000:1.0:1789760791.604550:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.604567:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5827, 5827] 00020000:00200000:1.0:1789760791.604576:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23867392 chunk: [23867392, 23871488) 23871488, 4096 00020000:00200000:1.0:1789760791.604581:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.604604:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5827, 5827] 00000008:00010000:1.0:1789760791.604633:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.604639:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.604652:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23867392, 23871488) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.604676:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23867392=0x16c3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.604743:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.604746:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23871488 00000020:00010000:1.0:1789760791.604754:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.604757:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.604760:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.604763:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.604766:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.604775:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.604782:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.604798:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760791.604905:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 23867392<>23871488 00000100:00100000:0.0:1789760791.605411:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760791.605433:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507383168:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.646291:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973147 gen 3 00000008:00010000:0.0:1789760791.646362:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760791.646371:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.646413:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.646441:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760791.646472:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507383168:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.647043:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5827, 5827] 00000080:00200000:1.0:1789760791.647062:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.647069:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.647087:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23871488 i_blocks 47111 00000020:00200000:1.0:1789760791.647091:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.647102:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.647147:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23871488, count: 4096 00000080:00200000:1.0:1789760791.647156:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23871488, bytes: 4096 00000080:00200000:1.0:1789760791.647172:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.647184:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.647186:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23871488, 23875584) 0 0 00000080:00200000:1.0:1789760791.647192:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.647209:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5828, 5828] 00020000:00200000:1.0:1789760791.647218:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23871488 chunk: [23871488, 23875584) 23875584, 4096 00020000:00200000:1.0:1789760791.647222:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.647244:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5828, 5828] 00000008:00010000:1.0:1789760791.647263:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.647270:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.647283:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23871488, 23875584) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.647297:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23871488=0x16c4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.647358:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.647361:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23875584 00000020:00010000:1.0:1789760791.647369:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.647372:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.647414:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.647418:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.647421:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.647431:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.647438:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.647457:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760791.647563:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 23871488<>23875584 00000100:00100000:0.0:1789760791.647784:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760791.647808:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383296:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.719270:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973148 gen 3 00000008:00010000:0.0:1789760791.719324:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760791.719332:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.719344:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.719366:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760791.719429:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383296:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.719784:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5828, 5828] 00000080:00200000:1.0:1789760791.719802:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.719808:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.719826:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23875584 i_blocks 47111 00000020:00200000:1.0:1789760791.719830:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.719841:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.719879:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23875584, count: 4096 00000080:00200000:1.0:1789760791.719886:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23875584, bytes: 4096 00000080:00200000:1.0:1789760791.719898:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.719908:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.719910:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23875584, 23879680) 0 0 00000080:00200000:1.0:1789760791.719914:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.719931:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5829, 5829] 00020000:00200000:1.0:1789760791.719941:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23875584 chunk: [23875584, 23879680) 23879680, 4096 00020000:00200000:1.0:1789760791.719945:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.719966:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5829, 5829] 00000008:00010000:1.0:1789760791.719984:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.719990:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.720012:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23875584, 23879680) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.720027:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23875584=0x16c5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.720214:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.720217:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23879680 00000020:00010000:1.0:1789760791.720226:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.720230:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.720233:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.720236:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.720239:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.720248:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.720255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.720273:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760791.720422:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 23875584<>23879680 00000100:00100000:0.0:1789760791.720521:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760791.720834:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507383424:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.772457:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973149 gen 3 00000008:00010000:0.0:1789760791.772662:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760791.772671:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.772682:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.772701:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760791.772729:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507383424:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.772820:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5829, 5829] 00000080:00200000:1.0:1789760791.772838:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.772844:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.772860:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23879680 i_blocks 47111 00000020:00200000:1.0:1789760791.772864:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.772875:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.773121:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23879680, count: 4096 00000080:00200000:1.0:1789760791.773130:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23879680, bytes: 4096 00000080:00200000:1.0:1789760791.773140:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.773149:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.773152:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23879680, 23883776) 0 0 00000080:00200000:1.0:1789760791.773157:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.773173:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5830, 5830] 00020000:00200000:1.0:1789760791.773181:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23879680 chunk: [23879680, 23883776) 23883776, 4096 00020000:00200000:1.0:1789760791.773186:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.773472:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5830, 5830] 00000008:00010000:1.0:1789760791.773492:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.773498:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.773511:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23879680, 23883776) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.773526:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23879680=0x16c6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.773587:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.773590:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23883776 00000020:00010000:1.0:1789760791.773669:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.773673:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.773677:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.773680:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.773683:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.773693:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.773700:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.773719:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760791.774119:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 23879680<>23883776 00000100:00100000:0.0:1789760791.774295:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760791.774320:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383552:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.822895:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973150 gen 3 00000008:00010000:0.0:1789760791.822949:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760791.822958:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.822970:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.822989:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760791.823020:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383552:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760791.823412:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5830, 5830] 00000080:00200000:1.0:1789760791.823429:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760791.823435:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760791.823451:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23883776 i_blocks 47111 00000020:00200000:1.0:1789760791.823455:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760791.823465:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760791.823500:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23883776, count: 4096 00000080:00200000:1.0:1789760791.823507:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23883776, bytes: 4096 00000080:00200000:1.0:1789760791.823578:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760791.823588:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760791.823591:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23883776, 23887872) 0 0 00000080:00200000:1.0:1789760791.823595:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760791.823621:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5831, 5831] 00020000:00200000:1.0:1789760791.823630:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23883776 chunk: [23883776, 23887872) 23887872, 4096 00020000:00200000:1.0:1789760791.823635:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760791.823710:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5831, 5831] 00000008:00010000:1.0:1789760791.823731:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760791.823738:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760791.823749:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23883776, 23887872) append 0 append lockpos 0 00000080:00200000:1.0:1789760791.823763:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23883776=0x16c7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760791.823825:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760791.823828:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23887872 00000020:00010000:1.0:1789760791.823836:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760791.823839:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.823842:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760791.823845:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760791.823847:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760791.823856:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760791.823863:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760791.823881:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760791.823986:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 23883776<>23887872 00000100:00100000:0.0:1789760791.824060:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383680:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.861616:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973151 gen 3 00000008:00010000:0.0:1789760791.861679:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760791.861688:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.861700:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.861772:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760791.861805:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383680:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760791.862308:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5831, 5831] 00000080:00200000:2.0:1789760791.862325:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760791.862331:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760791.862347:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23887872 i_blocks 47111 00000020:00200000:2.0:1789760791.862350:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760791.862360:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760791.862546:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23887872, count: 4096 00000080:00200000:2.0:1789760791.862554:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23887872, bytes: 4096 00000080:00200000:2.0:1789760791.862566:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760791.862575:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760791.862577:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23887872, 23891968) 0 0 00000080:00200000:2.0:1789760791.862583:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760791.862598:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5832, 5832] 00020000:00200000:2.0:1789760791.862606:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23887872 chunk: [23887872, 23891968) 23891968, 4096 00020000:00200000:2.0:1789760791.862620:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760791.862640:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5832, 5832] 00000008:00010000:2.0:1789760791.862709:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760791.862714:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760791.862725:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23887872, 23891968) append 0 append lockpos 0 00000080:00200000:2.0:1789760791.862738:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23887872=0x16c8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760791.862800:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760791.862803:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23891968 00000020:00010000:2.0:1789760791.862810:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760791.862813:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.862817:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.862820:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760791.862823:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760791.862831:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760791.862839:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760791.862860:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760791.863021:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 23887872<>23891968 00000080:00200000:0.0:1789760791.917474:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5832, 5832] 00000080:00200000:0.0:1789760791.917493:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760791.917499:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760791.917517:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23891968 i_blocks 47111 00000020:00200000:0.0:1789760791.917522:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760791.917533:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760791.917571:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23891968, count: 4096 00000080:00200000:0.0:1789760791.917578:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23891968, bytes: 4096 00000080:00200000:0.0:1789760791.917589:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760791.917599:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760791.917601:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23891968, 23896064) 0 0 00000080:00200000:0.0:1789760791.917606:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760791.917634:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5833, 5833] 00020000:00200000:0.0:1789760791.917653:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23891968 chunk: [23891968, 23896064) 23896064, 4096 00020000:00200000:0.0:1789760791.917658:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760791.917678:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5833, 5833] 00000008:00010000:0.0:1789760791.917697:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760791.917703:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760791.917713:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23891968, 23896064) append 0 append lockpos 0 00000080:00200000:0.0:1789760791.917729:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23891968=0x16c9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760791.917790:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760791.917793:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23896064 00000020:00010000:0.0:1789760791.917801:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760791.917805:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760791.917808:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760791.917812:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760791.917815:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760791.917824:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760791.917830:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760791.917848:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760791.917966:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23891968<>23896064 00000100:00100000:0.0:1789760791.918012:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760791.918031:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383936:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760791.979163:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973153 gen 3 00000008:00010000:0.0:1789760791.979217:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760791.979226:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760791.979240:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760791.979270:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760791.979302:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507383936:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760791.979426:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5833, 5833] 00000080:00200000:2.0:1789760791.979443:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760791.979449:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760791.979465:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23896064 i_blocks 47111 00000020:00200000:2.0:1789760791.979469:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760791.979479:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760791.979514:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23896064, count: 4096 00000080:00200000:2.0:1789760791.979522:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23896064, bytes: 4096 00000080:00200000:2.0:1789760791.979533:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760791.979542:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760791.979545:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23896064, 23900160) 0 0 00000080:00200000:2.0:1789760791.979550:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760791.979567:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5834, 5834] 00020000:00200000:2.0:1789760791.979576:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23896064 chunk: [23896064, 23900160) 23900160, 4096 00020000:00200000:2.0:1789760791.979581:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760791.979601:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5834, 5834] 00000008:00010000:2.0:1789760791.979633:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760791.979639:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760791.979725:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23896064, 23900160) append 0 append lockpos 0 00000080:00200000:2.0:1789760791.979741:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23896064=0x16ca000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760791.979806:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760791.979810:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23900160 00000020:00010000:2.0:1789760791.979818:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760791.979822:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.979825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760791.979829:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760791.979833:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760791.979841:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760791.979850:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760791.979867:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760791.980047:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 23896064<>23900160 00000080:00200000:2.0:1789760792.011852:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5834, 5834] 00000080:00200000:2.0:1789760792.011870:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.011876:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.011892:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23900160 i_blocks 47111 00000020:00200000:2.0:1789760792.011940:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.011952:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.011986:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23900160, count: 4096 00000080:00200000:2.0:1789760792.011994:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23900160, bytes: 4096 00000080:00200000:2.0:1789760792.012006:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.012017:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.012019:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23900160, 23904256) 0 0 00000080:00200000:2.0:1789760792.012024:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.012042:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5835, 5835] 00020000:00200000:2.0:1789760792.012113:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23900160 chunk: [23900160, 23904256) 23904256, 4096 00020000:00200000:2.0:1789760792.012119:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.012139:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5835, 5835] 00000008:00010000:2.0:1789760792.012158:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.012164:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.012177:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23900160, 23904256) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.012193:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23900160=0x16cb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.012252:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.012255:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23904256 00000020:00010000:2.0:1789760792.012263:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.012267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.012269:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.012274:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.012277:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.012285:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.012293:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.012315:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac6c0 00000008:00100000:2.0:1789760792.013185:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 23900160<>23904256 00000080:00200000:2.0:1789760792.071716:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5835, 5835] 00000080:00200000:2.0:1789760792.071748:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.071755:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.071775:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23904256 i_blocks 47111 00000020:00200000:2.0:1789760792.071780:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.071796:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.071834:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23904256, count: 4096 00000080:00200000:2.0:1789760792.071844:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23904256, bytes: 4096 00000080:00200000:2.0:1789760792.071861:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.071871:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.071873:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23904256, 23908352) 0 0 00000080:00200000:2.0:1789760792.071877:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.071894:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5836, 5836] 00020000:00200000:2.0:1789760792.071904:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23904256 chunk: [23904256, 23908352) 23908352, 4096 00020000:00200000:2.0:1789760792.071909:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.071933:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5836, 5836] 00000008:00010000:2.0:1789760792.071951:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.071957:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.071970:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23904256, 23908352) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.071986:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23904256=0x16cc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.072058:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.072060:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23908352 00000020:00010000:2.0:1789760792.072070:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.072073:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.072076:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.072080:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.072082:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.072090:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.072098:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.072128:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760792.072240:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 23904256<>23908352 00000080:00200000:2.0:1789760792.103027:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5836, 5836] 00000080:00200000:2.0:1789760792.103042:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.103047:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.103061:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23908352 i_blocks 47111 00000020:00200000:2.0:1789760792.103064:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.103072:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.103180:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23908352, count: 4096 00000080:00200000:2.0:1789760792.103187:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23908352, bytes: 4096 00000080:00200000:2.0:1789760792.103198:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.103206:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.103208:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23908352, 23912448) 0 0 00000080:00200000:2.0:1789760792.103212:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.103225:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5837, 5837] 00020000:00200000:2.0:1789760792.103233:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23908352 chunk: [23908352, 23912448) 23912448, 4096 00020000:00200000:2.0:1789760792.103236:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.103252:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5837, 5837] 00000008:00010000:2.0:1789760792.103266:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.103271:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.103279:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23908352, 23912448) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.103289:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23908352=0x16cd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.103327:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.103330:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23912448 00000020:00010000:2.0:1789760792.103417:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.103422:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.103425:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.103427:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.103430:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.103436:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.103442:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.144168:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5837, 5837] 00000080:00200000:2.0:1789760792.144185:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.144191:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.144209:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23912448 i_blocks 47111 00000020:00200000:2.0:1789760792.144213:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.144224:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.144260:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23912448, count: 4096 00000080:00200000:2.0:1789760792.144267:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23912448, bytes: 4096 00000080:00200000:2.0:1789760792.144278:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.144288:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.144290:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23912448, 23916544) 0 0 00000080:00200000:2.0:1789760792.144296:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.144314:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5838, 5838] 00020000:00200000:2.0:1789760792.144323:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23912448 chunk: [23912448, 23916544) 23916544, 4096 00020000:00200000:2.0:1789760792.144328:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.144348:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5838, 5838] 00000008:00010000:2.0:1789760792.144365:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.144372:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.144419:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23912448, 23916544) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.144434:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23912448=0x16ce000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.144499:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.144502:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23916544 00000020:00010000:2.0:1789760792.144511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.144515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.144518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.144521:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.144524:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.144532:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.144540:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.144558:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760792.144963:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 23912448<>23916544 00000080:00200000:2.0:1789760792.189173:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5838, 5838] 00000080:00200000:2.0:1789760792.189189:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.189196:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.189214:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23916544 i_blocks 47111 00000020:00200000:2.0:1789760792.189219:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.189229:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.189265:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23916544, count: 4096 00000080:00200000:2.0:1789760792.189274:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23916544, bytes: 4096 00000080:00200000:2.0:1789760792.189289:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.189300:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.189303:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23916544, 23920640) 0 0 00000080:00200000:2.0:1789760792.189308:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.189327:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5839, 5839] 00020000:00200000:2.0:1789760792.189336:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23916544 chunk: [23916544, 23920640) 23920640, 4096 00020000:00200000:2.0:1789760792.189341:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.189367:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5839, 5839] 00000008:00010000:2.0:1789760792.189416:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.189424:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.189438:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23916544, 23920640) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.189453:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23916544=0x16cf000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.189521:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.189524:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23920640 00000020:00010000:2.0:1789760792.189533:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.189537:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.189540:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.189543:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.189546:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.189553:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.189560:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.189578:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760792.189701:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 23916544<>23920640 00000080:00200000:2.0:1789760792.237017:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5839, 5839] 00000080:00200000:2.0:1789760792.237034:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.237040:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.237129:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23920640 i_blocks 47111 00000020:00200000:2.0:1789760792.237133:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.237143:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.237177:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23920640, count: 4096 00000080:00200000:2.0:1789760792.237185:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23920640, bytes: 4096 00000080:00200000:2.0:1789760792.237196:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.237206:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.237208:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23920640, 23924736) 0 0 00000080:00200000:2.0:1789760792.237213:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.237229:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5840, 5840] 00020000:00200000:2.0:1789760792.237237:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23920640 chunk: [23920640, 23924736) 23924736, 4096 00020000:00200000:2.0:1789760792.237241:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.237261:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5840, 5840] 00000008:00010000:2.0:1789760792.237278:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.237285:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.237295:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23920640, 23924736) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.237309:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23920640=0x16d0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.237368:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.237371:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23924736 00000020:00010000:2.0:1789760792.238030:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.238037:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.238040:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.238045:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.238047:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.238232:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.238241:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.238261:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac300 00000008:00100000:2.0:1789760792.238405:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 23920640<>23924736 00000080:00200000:2.0:1789760792.325326:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5840, 5840] 00000080:00200000:2.0:1789760792.325341:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.325345:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.325360:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23924736 i_blocks 47111 00000020:00200000:2.0:1789760792.325363:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.325371:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.325837:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23924736, count: 4096 00000080:00200000:2.0:1789760792.325847:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23924736, bytes: 4096 00000080:00200000:2.0:1789760792.325858:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.325867:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.325870:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23924736, 23928832) 0 0 00000080:00200000:2.0:1789760792.325875:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.325893:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5841, 5841] 00020000:00200000:2.0:1789760792.325902:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23924736 chunk: [23924736, 23928832) 23928832, 4096 00020000:00200000:2.0:1789760792.325906:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.325926:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5841, 5841] 00000008:00010000:2.0:1789760792.325947:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.325955:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.325968:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23924736, 23928832) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.325982:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23924736=0x16d1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.326045:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.326048:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23928832 00000020:00010000:2.0:1789760792.326056:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.326060:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.326063:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.326067:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.326070:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.326079:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.326087:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.326107:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760792.326215:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 23924736<>23928832 00010000:00010000:0.0:1789760792.384585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000008:00010000:2.0:1789760792.411368:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760792.411410:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760792.411424:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760792.411456:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760792.411499:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507384960:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760792.411650:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5841, 5841] 00000080:00200000:0.0:1789760792.411681:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760792.411688:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760792.411706:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23928832 i_blocks 47111 00000020:00200000:0.0:1789760792.411711:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760792.411723:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760792.411758:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23928832, count: 4096 00000080:00200000:0.0:1789760792.411764:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23928832, bytes: 4096 00000080:00200000:0.0:1789760792.411781:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760792.411791:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760792.411793:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23928832, 23932928) 0 0 00000080:00200000:0.0:1789760792.411798:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760792.411815:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5842, 5842] 00020000:00200000:0.0:1789760792.411825:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23928832 chunk: [23928832, 23932928) 23932928, 4096 00020000:00200000:0.0:1789760792.411829:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760792.411850:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5842, 5842] 00000008:00010000:0.0:1789760792.411868:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760792.411874:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760792.411887:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23928832, 23932928) append 0 append lockpos 0 00000080:00200000:0.0:1789760792.411903:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23928832=0x16d2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760792.411969:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760792.411972:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23932928 00000020:00010000:0.0:1789760792.411981:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760792.411985:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760792.411988:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760792.411992:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760792.411995:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760792.412004:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760792.412011:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760792.412028:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760792.412152:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 23928832<>23932928 00000100:00100000:1.0:1789760792.412423:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507385088:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760792.460076:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973161 gen 3 00000100:00100000:0.0:1789760792.460122:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25bbb80 x1876699507385088/t4294973162(4294973162) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760808 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760792.460172:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760792.460180:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760792.460192:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760792.460218:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760792.460245:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507385088:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760792.460421:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5842, 5842] 00000080:00200000:2.0:1789760792.460440:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.460446:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.460462:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23932928 i_blocks 47111 00000020:00200000:2.0:1789760792.460466:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.460476:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.460521:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23932928, count: 4096 00000080:00200000:2.0:1789760792.460530:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23932928, bytes: 4096 00000080:00200000:2.0:1789760792.460541:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.460551:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.460554:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23932928, 23937024) 0 0 00000080:00200000:2.0:1789760792.460559:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.460576:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5843, 5843] 00020000:00200000:2.0:1789760792.460585:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23932928 chunk: [23932928, 23937024) 23937024, 4096 00020000:00200000:2.0:1789760792.460590:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.460621:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5843, 5843] 00000008:00010000:2.0:1789760792.460862:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.460870:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.460884:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23932928, 23937024) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.460910:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23932928=0x16d3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.460968:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.460971:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23937024 00000020:00010000:2.0:1789760792.460980:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.460983:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.460987:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.460990:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.460992:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.461000:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.461007:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760792.461179:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:0.0:1789760792.461355:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9c00 - object 0x280000401:3 offset 23932928<>23937024 00000100:00100000:0.0:1789760792.461407:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760792.461425:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507385216:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760792.510214:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973163 gen 3 00000008:00010000:0.0:1789760792.510352:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:0.0:1789760792.510361:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760792.510373:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760792.510437:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:0.0:1789760792.510465:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507385216:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760792.510619:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5843, 5843] 00000080:00200000:2.0:1789760792.510638:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.510645:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.510789:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23937024 i_blocks 47111 00000020:00200000:2.0:1789760792.510795:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.510806:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.510844:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23937024, count: 4096 00000080:00200000:2.0:1789760792.510852:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23937024, bytes: 4096 00000080:00200000:2.0:1789760792.510864:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.510873:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.510876:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23937024, 23941120) 0 0 00000080:00200000:2.0:1789760792.510881:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.510908:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5844, 5844] 00020000:00200000:2.0:1789760792.510918:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23937024 chunk: [23937024, 23941120) 23941120, 4096 00020000:00200000:2.0:1789760792.510922:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.510943:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5844, 5844] 00000008:00010000:2.0:1789760792.510963:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.510969:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.510982:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23937024, 23941120) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.510996:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23937024=0x16d4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.511123:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.511126:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23941120 00000020:00010000:2.0:1789760792.511136:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.511139:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.511143:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.511146:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.511150:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.511158:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.511165:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.511183:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac6c0 00000008:00100000:2.0:1789760792.511993:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 23937024<>23941120 00000080:00200000:2.0:1789760792.562418:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5844, 5844] 00000080:00200000:2.0:1789760792.562435:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.562441:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.562456:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23941120 i_blocks 47111 00000020:00200000:2.0:1789760792.562459:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.562469:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.562502:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23941120, count: 4096 00000080:00200000:2.0:1789760792.562518:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23941120, bytes: 4096 00000080:00200000:2.0:1789760792.562528:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.562536:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.562538:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23941120, 23945216) 0 0 00000080:00200000:2.0:1789760792.562543:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.562558:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5845, 5845] 00020000:00200000:2.0:1789760792.562565:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23941120 chunk: [23941120, 23945216) 23945216, 4096 00020000:00200000:2.0:1789760792.562569:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.562587:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5845, 5845] 00000008:00010000:2.0:1789760792.562610:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.562616:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.562627:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23941120, 23945216) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.562640:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23941120=0x16d5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.562799:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.562802:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23945216 00000020:00010000:2.0:1789760792.562811:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.562815:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.562817:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.562821:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.562825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.562832:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.562838:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.562853:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760792.562951:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 23941120<>23945216 00000080:00200000:2.0:1789760792.606372:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5845, 5845] 00000080:00200000:2.0:1789760792.606423:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.606430:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.606445:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23945216 i_blocks 47111 00000020:00200000:2.0:1789760792.606449:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.606459:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.606491:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23945216, count: 4096 00000080:00200000:2.0:1789760792.606498:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23945216, bytes: 4096 00000080:00200000:2.0:1789760792.606508:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.606518:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.606520:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23945216, 23949312) 0 0 00000080:00200000:2.0:1789760792.606524:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.606540:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5846, 5846] 00020000:00200000:2.0:1789760792.606548:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23945216 chunk: [23945216, 23949312) 23949312, 4096 00020000:00200000:2.0:1789760792.606553:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.606571:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5846, 5846] 00000008:00010000:2.0:1789760792.606587:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.606593:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.606615:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23945216, 23949312) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.606629:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23945216=0x16d6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.606810:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.606813:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23949312 00000020:00010000:2.0:1789760792.606822:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.606826:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.606829:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.606832:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.606836:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.606845:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.606851:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.606868:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760792.606976:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022d80 - object 0x280000401:3 offset 23945216<>23949312 00000080:00200000:0.0:1789760792.646528:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5846, 5846] 00000080:00200000:0.0:1789760792.646545:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760792.646552:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760792.646566:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23949312 i_blocks 47111 00000020:00200000:0.0:1789760792.646570:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760792.646580:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760792.646624:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23949312, count: 4096 00000080:00200000:0.0:1789760792.646631:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23949312, bytes: 4096 00000080:00200000:0.0:1789760792.646640:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760792.646649:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760792.646651:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23949312, 23953408) 0 0 00000080:00200000:0.0:1789760792.646664:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760792.646795:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5847, 5847] 00020000:00200000:0.0:1789760792.646803:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23949312 chunk: [23949312, 23953408) 23953408, 4096 00020000:00200000:0.0:1789760792.646808:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760792.646828:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5847, 5847] 00000008:00010000:0.0:1789760792.646845:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760792.646851:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760792.646862:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23949312, 23953408) append 0 append lockpos 0 00000080:00200000:0.0:1789760792.646877:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23949312=0x16d7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760792.646935:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760792.646938:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23953408 00000020:00010000:0.0:1789760792.646946:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760792.646950:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760792.646953:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760792.646957:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760792.646960:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760792.646968:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760792.646975:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760792.646990:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760792.647099:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9c00 - object 0x280000401:3 offset 23949312<>23953408 00000100:00100000:1.0:1789760792.647251:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507385728:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760792.691480:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973167 gen 3 00000008:00010000:0.0:1789760792.691531:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760792.691539:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760792.691551:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760792.691621:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760792.691665:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507385728:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760792.691953:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5847, 5847] 00000080:00200000:2.0:1789760792.691970:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.691977:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.691994:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23953408 i_blocks 47111 00000020:00200000:2.0:1789760792.692002:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.692014:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.692049:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23953408, count: 4096 00000080:00200000:2.0:1789760792.692056:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23953408, bytes: 4096 00000080:00200000:2.0:1789760792.692069:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.692079:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.692081:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23953408, 23957504) 0 0 00000080:00200000:2.0:1789760792.692085:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.692101:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5848, 5848] 00020000:00200000:2.0:1789760792.692110:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23953408 chunk: [23953408, 23957504) 23957504, 4096 00020000:00200000:2.0:1789760792.692114:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.692185:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5848, 5848] 00000008:00010000:2.0:1789760792.692204:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.692210:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.692222:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23953408, 23957504) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.692237:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23953408=0x16d8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.692301:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.692304:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23957504 00000020:00010000:2.0:1789760792.692312:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.692316:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.692320:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.692323:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.692326:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.692334:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.692342:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.692361:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760792.692494:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 23953408<>23957504 00000080:00200000:2.0:1789760792.732201:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5848, 5848] 00000080:00200000:2.0:1789760792.732217:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.732223:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.732239:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23957504 i_blocks 47111 00000020:00200000:2.0:1789760792.732242:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.732252:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.732287:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23957504, count: 4096 00000080:00200000:2.0:1789760792.732294:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23957504, bytes: 4096 00000080:00200000:2.0:1789760792.732304:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.732313:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.732315:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23957504, 23961600) 0 0 00000080:00200000:2.0:1789760792.732319:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.732334:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5849, 5849] 00020000:00200000:2.0:1789760792.732343:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23957504 chunk: [23957504, 23961600) 23961600, 4096 00020000:00200000:2.0:1789760792.732347:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.732364:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5849, 5849] 00000008:00010000:2.0:1789760792.732412:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.732418:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.732430:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23957504, 23961600) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.732443:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23957504=0x16d9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.732498:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.732501:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23961600 00000020:00010000:2.0:1789760792.732508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.732512:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.732515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.732518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.732522:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.732529:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.732536:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.732552:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760792.732786:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23957504<>23961600 00000080:00200000:2.0:1789760792.771150:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5849, 5849] 00000080:00200000:2.0:1789760792.771168:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760792.771174:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760792.771190:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23961600 i_blocks 47111 00000020:00200000:2.0:1789760792.771194:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760792.771206:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760792.771239:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23961600, count: 4096 00000080:00200000:2.0:1789760792.771245:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23961600, bytes: 4096 00000080:00200000:2.0:1789760792.771257:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.771265:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.771268:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23961600, 23965696) 0 0 00000080:00200000:2.0:1789760792.771272:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.771290:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5850, 5850] 00020000:00200000:2.0:1789760792.771299:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23961600 chunk: [23961600, 23965696) 23965696, 4096 00020000:00200000:2.0:1789760792.771304:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.771324:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5850, 5850] 00000008:00010000:2.0:1789760792.771342:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.771349:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.771361:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23961600, 23965696) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.771408:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23961600=0x16da000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.771468:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.771471:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23965696 00000020:00010000:2.0:1789760792.771480:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.771484:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.771487:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.771491:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be94e8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.771494:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.771501:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.771508:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.771526:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760792.771838:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23961600<>23965696 00000100:00100000:2.0:1789760792.813445:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973170 gen 3 00000008:00010000:2.0:1789760792.813490:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50accc0 00000020:00200000:2.0:1789760792.813497:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760792.813506:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760792.813521:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50accc0 00000100:00100000:2.0:1789760792.813549:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507386112:192.168.203.141@tcp:4:dd.0 00000080:00200008:2.0:1789760792.813831:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23965696, count: 4096 00000080:00200000:2.0:1789760792.813838:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23965696, bytes: 4096 00000080:00200000:2.0:1789760792.813848:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760792.813854:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760792.813857:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23965696, 23969792) 0 0 00000080:00200000:2.0:1789760792.813860:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760792.813875:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5851, 5851] 00020000:00200000:2.0:1789760792.813882:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23965696 chunk: [23965696, 23969792) 23969792, 4096 00020000:00200000:2.0:1789760792.813886:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760792.813929:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5851, 5851] 00000008:00010000:2.0:1789760792.813946:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760792.813951:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760792.813961:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23965696, 23969792) append 0 append lockpos 0 00000080:00200000:2.0:1789760792.813976:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23965696=0x16db000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760792.814034:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760792.814036:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23969792 00000020:00010000:2.0:1789760792.814045:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760792.814049:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.814052:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760792.814057:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760792.814059:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760792.814066:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760792.814072:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760792.814101:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760792.814262:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23965696<>23969792 00000100:00100000:2.0:1789760792.814305:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760792.814323:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507386240:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760792.860793:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973171 gen 3 00000008:00010000:2.0:1789760792.860846:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50acc00 00000020:00200000:2.0:1789760792.860854:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760792.860867:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760792.861146:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50acc00 00000100:00100000:2.0:1789760792.861179:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507386240:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760792.861267:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5851, 5851] 00000080:00200000:0.0:1789760792.861284:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760792.861289:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760792.861306:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23969792 i_blocks 47111 00000020:00200000:0.0:1789760792.861309:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760792.861319:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760792.861453:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23969792, count: 4096 00000080:00200000:0.0:1789760792.861462:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23969792, bytes: 4096 00000080:00200000:0.0:1789760792.861473:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760792.861482:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760792.861484:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23969792, 23973888) 0 0 00000080:00200000:0.0:1789760792.861490:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760792.861507:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5852, 5852] 00020000:00200000:0.0:1789760792.861515:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23969792 chunk: [23969792, 23973888) 23973888, 4096 00020000:00200000:0.0:1789760792.861520:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760792.861538:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5852, 5852] 00000008:00010000:0.0:1789760792.861555:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760792.861561:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760792.861573:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23969792, 23973888) append 0 append lockpos 0 00000080:00200000:0.0:1789760792.861585:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23969792=0x16dc000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760792.861714:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760792.861717:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23973888 00000020:00010000:0.0:1789760792.861726:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:0.0:1789760792.861730:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760792.861733:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760792.861736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760792.861739:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760792.861748:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760792.861755:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760792.861774:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760792.861884:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9c00 - object 0x280000401:3 offset 23969792<>23973888 00000100:00100000:0.0:1789760792.862238:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760792.862257:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507386368:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760792.923826:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760792.923842:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760792.923854:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760792.923878:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760792.923905:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507386368:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760792.924771:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5852, 5852] 00000080:00200000:1.0:1789760792.924790:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760792.924797:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760792.924812:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23973888 i_blocks 47111 00000020:00200000:1.0:1789760792.924816:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760792.924827:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760792.924862:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23973888, count: 4096 00000080:00200000:1.0:1789760792.924870:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23973888, bytes: 4096 00000080:00200000:1.0:1789760792.924881:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760792.924890:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760792.924892:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23973888, 23977984) 0 0 00000080:00200000:1.0:1789760792.924897:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760792.924913:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5853, 5853] 00020000:00200000:1.0:1789760792.924922:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23973888 chunk: [23973888, 23977984) 23977984, 4096 00020000:00200000:1.0:1789760792.924927:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760792.924947:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5853, 5853] 00000008:00010000:1.0:1789760792.924965:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760792.924971:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760792.924982:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23973888, 23977984) append 0 append lockpos 0 00000080:00200000:1.0:1789760792.924995:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23973888=0x16dd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760792.925057:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760792.925060:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23977984 00000020:00010000:1.0:1789760792.925068:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760792.925071:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760792.925075:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760792.925079:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760792.925081:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760792.925089:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760792.925096:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760792.925113:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451f00 00000008:00100000:1.0:1789760792.925218:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 23973888<>23977984 00000100:00100000:0.0:1789760792.925293:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507386496:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760792.959460:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973172 gen 3 00000100:00100000:0.0:1789760792.959500:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8168000 x1876699507386496/t4294973173(4294973173) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760808 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760792.959548:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451f00 00000020:00200000:0.0:1789760792.959555:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760792.959565:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760792.959637:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451f00 00000100:00100000:0.0:1789760792.959672:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507386496:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760792.963347:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5853, 5853] 00000080:00200000:1.0:1789760792.963711:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760792.963720:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760792.963742:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23977984 i_blocks 47111 00000020:00200000:1.0:1789760792.963748:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760792.963767:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760792.963805:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23977984, count: 4096 00000080:00200000:1.0:1789760792.963815:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23977984, bytes: 4096 00000080:00200000:1.0:1789760792.963830:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760792.963840:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760792.963843:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23977984, 23982080) 0 0 00000080:00200000:1.0:1789760792.963848:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760792.963866:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5854, 5854] 00020000:00200000:1.0:1789760792.963875:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23977984 chunk: [23977984, 23982080) 23982080, 4096 00020000:00200000:1.0:1789760792.963879:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760792.963902:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5854, 5854] 00000008:00010000:1.0:1789760792.963923:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760792.963929:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760792.964740:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23977984, 23982080) append 0 append lockpos 0 00000080:00200000:1.0:1789760792.964762:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23977984=0x16de000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760792.964840:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760792.964844:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23982080 00000020:00010000:1.0:1789760792.964853:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760792.964857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760792.964861:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760792.964865:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760792.964868:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760792.964879:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760792.964887:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760792.964920:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451d80 00000008:00100000:1.0:1789760792.965181:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 23977984<>23982080 00000100:00100000:0.0:1789760792.965281:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507386624:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.012963:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973174 gen 3 00000008:00010000:0.0:1789760793.013023:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451d80 00000020:00200000:0.0:1789760793.013031:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.013040:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.013113:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451d80 00000100:00100000:0.0:1789760793.013141:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507386624:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760793.014786:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5854, 5854] 00000080:00200000:2.0:1789760793.014816:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760793.014824:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760793.014841:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23982080 i_blocks 47111 00000020:00200000:2.0:1789760793.014846:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760793.014864:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760793.014902:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23982080, count: 4096 00000080:00200000:2.0:1789760793.014910:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23982080, bytes: 4096 00000080:00200000:2.0:1789760793.014925:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760793.014934:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760793.014936:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23982080, 23986176) 0 0 00000080:00200000:2.0:1789760793.014940:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760793.014959:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5855, 5855] 00020000:00200000:2.0:1789760793.014967:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23982080 chunk: [23982080, 23986176) 23986176, 4096 00020000:00200000:2.0:1789760793.014971:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760793.014993:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5855, 5855] 00000008:00010000:2.0:1789760793.015014:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760793.015021:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760793.015034:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23982080, 23986176) append 0 append lockpos 0 00000080:00200000:2.0:1789760793.015050:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23982080=0x16df000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760793.015122:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760793.015125:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23986176 00000020:00010000:2.0:1789760793.015134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760793.015137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.015141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.015144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760793.015147:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760793.015156:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760793.015163:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760793.015194:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760793.015307:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020e00 - object 0x280000401:3 offset 23982080<>23986176 00000080:00200000:0.0:1789760793.056083:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5855, 5855] 00000080:00200000:0.0:1789760793.056099:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760793.056105:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760793.056119:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23986176 i_blocks 47111 00000020:00200000:0.0:1789760793.056124:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760793.056133:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760793.056166:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23986176, count: 4096 00000080:00200000:0.0:1789760793.056174:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23986176, bytes: 4096 00000080:00200000:0.0:1789760793.056183:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760793.056191:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760793.056193:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23986176, 23990272) 0 0 00000080:00200000:0.0:1789760793.056198:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760793.056215:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5856, 5856] 00020000:00200000:0.0:1789760793.056223:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23986176 chunk: [23986176, 23990272) 23990272, 4096 00020000:00200000:0.0:1789760793.056229:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760793.056248:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5856, 5856] 00000008:00010000:0.0:1789760793.056267:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760793.056273:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760793.056284:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23986176, 23990272) append 0 append lockpos 0 00000080:00200000:0.0:1789760793.056298:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23986176=0x16e0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760793.056352:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760793.056355:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23990272 00000020:00010000:0.0:1789760793.056363:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:0.0:1789760793.056366:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760793.056369:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760793.056373:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760793.056416:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760793.056426:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760793.056432:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760793.056448:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760793.056554:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9c00 - object 0x280000401:3 offset 23986176<>23990272 00000100:00100000:0.0:1789760793.056589:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760793.056616:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507386880:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.091036:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973176 gen 3 00000008:00010000:0.0:1789760793.091112:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760793.091121:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.091132:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.091158:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760793.091200:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507386880:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760793.091555:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5856, 5856] 00000080:00200000:2.0:1789760793.091572:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760793.091579:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760793.091596:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23990272 i_blocks 47111 00000020:00200000:2.0:1789760793.091639:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760793.091650:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760793.091683:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23990272, count: 4096 00000080:00200000:2.0:1789760793.091694:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23990272, bytes: 4096 00000080:00200000:2.0:1789760793.091706:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760793.091716:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760793.091718:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23990272, 23994368) 0 0 00000080:00200000:2.0:1789760793.091722:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760793.091744:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5857, 5857] 00020000:00200000:2.0:1789760793.091753:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23990272 chunk: [23990272, 23994368) 23994368, 4096 00020000:00200000:2.0:1789760793.091759:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760793.091779:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5857, 5857] 00000008:00010000:2.0:1789760793.091798:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760793.091805:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760793.091816:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23990272, 23994368) append 0 append lockpos 0 00000080:00200000:2.0:1789760793.091830:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23990272=0x16e1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760793.091888:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760793.091891:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23994368 00000020:00010000:2.0:1789760793.092006:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760793.092011:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.092013:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.092017:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760793.092019:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760793.092027:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760793.092034:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760793.092063:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760793.092167:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 23990272<>23994368 00000100:00100000:2.0:1789760793.133683:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973177 gen 3 00000008:00010000:2.0:1789760793.133733:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760793.133741:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760793.133751:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760793.133777:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760793.133807:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507387008:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760793.133986:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5857, 5857] 00000080:00200000:0.0:1789760793.134004:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760793.134009:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760793.134025:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23994368 i_blocks 47111 00000020:00200000:0.0:1789760793.134030:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760793.134040:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760793.134134:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23994368, count: 4096 00000080:00200000:0.0:1789760793.134142:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23994368, bytes: 4096 00000080:00200000:0.0:1789760793.134155:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760793.134163:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760793.134166:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23994368, 23998464) 0 0 00000080:00200000:0.0:1789760793.134170:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760793.134187:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5858, 5858] 00020000:00200000:0.0:1789760793.134195:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23994368 chunk: [23994368, 23998464) 23998464, 4096 00020000:00200000:0.0:1789760793.134200:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760793.134220:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5858, 5858] 00000008:00010000:0.0:1789760793.134237:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760793.134242:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760793.134253:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23994368, 23998464) append 0 append lockpos 0 00000080:00200000:0.0:1789760793.134267:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23994368=0x16e2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760793.134326:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760793.134329:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 23998464 00000020:00010000:0.0:1789760793.134338:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:0.0:1789760793.134342:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760793.134345:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760793.134348:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760793.134352:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760793.134530:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760793.134538:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760793.134558:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760793.134801:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9c00 - object 0x280000401:3 offset 23994368<>23998464 00000100:00100000:1.0:1789760793.134874:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507387136:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.167532:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973178 gen 3 00000008:00010000:0.0:1789760793.167584:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760793.167593:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.167614:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.167637:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760793.167668:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507387136:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760793.168006:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5858, 5858] 00000080:00200000:2.0:1789760793.168023:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760793.168030:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760793.168056:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 23998464 i_blocks 47111 00000020:00200000:2.0:1789760793.168060:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760793.168070:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760793.168103:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 23998464, count: 4096 00000080:00200000:2.0:1789760793.168110:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 23998464, bytes: 4096 00000080:00200000:2.0:1789760793.168122:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760793.168131:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760793.168134:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [23998464, 24002560) 0 0 00000080:00200000:2.0:1789760793.168138:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760793.168154:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5859, 5859] 00020000:00200000:2.0:1789760793.168163:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 23998464 chunk: [23998464, 24002560) 24002560, 4096 00020000:00200000:2.0:1789760793.168167:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760793.168186:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5859, 5859] 00000008:00010000:2.0:1789760793.168204:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760793.168211:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760793.168222:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [23998464, 24002560) append 0 append lockpos 0 00000080:00200000:2.0:1789760793.168236:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=23998464=0x16e3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760793.168297:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760793.168300:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24002560 00000020:00010000:2.0:1789760793.168309:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760793.168312:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.168315:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.168319:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760793.168322:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760793.168328:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760793.168335:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760793.168354:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760793.168516:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 23998464<>24002560 00000100:00100000:2.0:1789760793.168563:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760793.168582:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507387264:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760793.203820:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973179 gen 3 00000008:00010000:2.0:1789760793.203875:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760793.203884:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760793.203895:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760793.203921:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760793.203952:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507387264:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760793.204063:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5859, 5859] 00000080:00200000:0.0:1789760793.204081:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760793.204086:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760793.204102:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24002560 i_blocks 47111 00000020:00200000:0.0:1789760793.204106:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760793.204117:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760793.204152:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24002560, count: 4096 00000080:00200000:0.0:1789760793.204159:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24002560, bytes: 4096 00000080:00200000:0.0:1789760793.204170:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760793.204180:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760793.204182:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24002560, 24006656) 0 0 00000080:00200000:0.0:1789760793.204186:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760793.204203:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5860, 5860] 00020000:00200000:0.0:1789760793.204211:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24002560 chunk: [24002560, 24006656) 24006656, 4096 00020000:00200000:0.0:1789760793.204216:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760793.204235:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5860, 5860] 00000008:00010000:0.0:1789760793.204252:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760793.204258:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760793.204270:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24002560, 24006656) append 0 append lockpos 0 00000080:00200000:0.0:1789760793.204284:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24002560=0x16e4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760793.204342:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760793.204346:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24006656 00000020:00010000:0.0:1789760793.204355:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:0.0:1789760793.204521:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760793.204524:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760793.204529:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760793.204532:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760793.204541:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760793.204549:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760793.204567:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:0.0:1789760793.204692:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9c00 - object 0x280000401:3 offset 24002560<>24006656 00000100:00100000:0.0:1789760793.204728:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760793.204745:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507387392:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.267471:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973180 gen 3 00000008:00010000:0.0:1789760793.267518:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:0.0:1789760793.267527:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.267540:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.267563:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000100:00100000:0.0:1789760793.267594:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507387392:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.267764:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5860, 5860] 00000080:00200000:1.0:1789760793.267782:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.267789:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.267805:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24006656 i_blocks 47111 00000020:00200000:1.0:1789760793.267809:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.267821:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.267858:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24006656, count: 4096 00000080:00200000:1.0:1789760793.267865:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24006656, bytes: 4096 00000080:00200000:1.0:1789760793.267879:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.267889:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.267892:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24006656, 24010752) 0 0 00000080:00200000:1.0:1789760793.267906:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.267926:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5861, 5861] 00020000:00200000:1.0:1789760793.267937:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24006656 chunk: [24006656, 24010752) 24010752, 4096 00020000:00200000:1.0:1789760793.267943:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.267966:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5861, 5861] 00000008:00010000:1.0:1789760793.267988:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.267995:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.268009:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24006656, 24010752) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.268025:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24006656=0x16e5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.268114:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.268117:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24010752 00000020:00010000:1.0:1789760793.268127:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.268131:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.268134:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.268137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.268141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.268151:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.268159:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.268178:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14513c0 00000008:00100000:1.0:1789760793.268292:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816b480 - object 0x280000401:3 offset 24006656<>24010752 00000100:00100000:1.0:1789760793.268329:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760793.268347:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507387520:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.324640:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973181 gen 3 00000008:00010000:0.0:1789760793.324802:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14513c0 00000020:00200000:0.0:1789760793.324810:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.324822:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.324845:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14513c0 00000100:00100000:0.0:1789760793.324875:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507387520:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.324997:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5861, 5861] 00000080:00200000:1.0:1789760793.325013:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.325019:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.325035:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24010752 i_blocks 47111 00000020:00200000:1.0:1789760793.325050:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.325060:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.325095:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24010752, count: 4096 00000080:00200000:1.0:1789760793.325102:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24010752, bytes: 4096 00000080:00200000:1.0:1789760793.325113:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.325122:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.325124:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24010752, 24014848) 0 0 00000080:00200000:1.0:1789760793.325128:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.325144:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5862, 5862] 00020000:00200000:1.0:1789760793.325153:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24010752 chunk: [24010752, 24014848) 24014848, 4096 00020000:00200000:1.0:1789760793.325157:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.325177:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5862, 5862] 00000008:00010000:1.0:1789760793.325194:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.325199:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.325212:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24010752, 24014848) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.325225:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24010752=0x16e6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.325282:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.325285:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24014848 00000020:00010000:1.0:1789760793.325294:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.325297:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.325300:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.325303:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.325306:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.325313:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.325320:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.325510:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451780 00000008:00100000:1.0:1789760793.325621:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816ad80 - object 0x280000401:3 offset 24010752<>24014848 00000100:00100000:0.0:1789760793.325988:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507387648:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.340209:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507387904:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.340324:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507388160:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.340368:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [0->1] 00000100:00100000:0.0:1789760793.340374:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2417:1876699507387776:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.340440:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388032:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.340463:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507388288:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.345021:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2417:1876699507387776:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.345115:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388032:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.345131:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507388288:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.345143:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507387904:192.168.203.141@tcp:400:kworker.0 00000100:00100000:0.0:1789760793.345154:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2417:1876699507388160:192.168.203.141@tcp:400:kworker.0 00000008:00010000:0.0:1789760793.379153:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451780 00000020:00200000:0.0:1789760793.379167:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.379178:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.379201:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451780 00000100:00100000:0.0:1789760793.379231:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507387648:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.379473:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5862, 5862] 00000080:00200000:1.0:1789760793.379490:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.379497:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.379512:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24014848 i_blocks 47111 00000020:00200000:1.0:1789760793.379516:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.379527:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.379561:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24014848, count: 4096 00000080:00200000:1.0:1789760793.379569:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24014848, bytes: 4096 00000080:00200000:1.0:1789760793.379579:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.379588:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.379590:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24014848, 24018944) 0 0 00000080:00200000:1.0:1789760793.379595:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.379622:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5863, 5863] 00020000:00200000:1.0:1789760793.379630:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24014848 chunk: [24014848, 24018944) 24018944, 4096 00020000:00200000:1.0:1789760793.379642:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.379732:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5863, 5863] 00000008:00010000:1.0:1789760793.379752:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.379757:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.379770:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24014848, 24018944) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.379786:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24014848=0x16e7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.379845:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.379848:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24018944 00000020:00010000:1.0:1789760793.379857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.379861:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.379864:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.379867:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.379870:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.379878:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.379885:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.379914:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451480 00000008:00100000:1.0:1789760793.380019:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24014848<>24018944 00000100:00100000:0.0:1789760793.380302:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388416:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.420654:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973183 gen 3 00000008:00010000:0.0:1789760793.420727:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451480 00000020:00200000:0.0:1789760793.420735:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.420745:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.420765:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451480 00000100:00100000:0.0:1789760793.420793:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388416:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.420942:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5863, 5863] 00000080:00200000:1.0:1789760793.420959:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.420965:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.420980:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24018944 i_blocks 47111 00000020:00200000:1.0:1789760793.420983:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.420992:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.421026:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24018944, count: 4096 00000080:00200000:1.0:1789760793.421033:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24018944, bytes: 4096 00000080:00200000:1.0:1789760793.421053:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.421063:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.421065:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24018944, 24023040) 0 0 00000080:00200000:1.0:1789760793.421070:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.421086:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5864, 5864] 00020000:00200000:1.0:1789760793.421095:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24018944 chunk: [24018944, 24023040) 24023040, 4096 00020000:00200000:1.0:1789760793.421100:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.421118:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5864, 5864] 00000008:00010000:1.0:1789760793.421137:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.421143:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.421156:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24018944, 24023040) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.421171:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24018944=0x16e8000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.421229:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.421232:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24023040 00000020:00010000:1.0:1789760793.421241:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.421245:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.421248:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.421251:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.421254:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.421261:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.421268:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.421286:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451240 00000008:00100000:1.0:1789760793.421529:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 24018944<>24023040 00000100:00100000:0.0:1789760793.422069:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760793.422092:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388544:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.463757:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973184 gen 3 00000008:00010000:0.0:1789760793.463808:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451240 00000020:00200000:0.0:1789760793.463817:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.463829:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.463850:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451240 00000100:00100000:0.0:1789760793.463878:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388544:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.464875:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5864, 5864] 00000080:00200000:1.0:1789760793.464906:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.464913:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.464929:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24023040 i_blocks 47111 00000020:00200000:1.0:1789760793.464934:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.464945:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.464980:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24023040, count: 4096 00000080:00200000:1.0:1789760793.464987:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24023040, bytes: 4096 00000080:00200000:1.0:1789760793.464998:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.465008:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.465010:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24023040, 24027136) 0 0 00000080:00200000:1.0:1789760793.465014:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.465032:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5865, 5865] 00020000:00200000:1.0:1789760793.465040:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24023040 chunk: [24023040, 24027136) 24027136, 4096 00020000:00200000:1.0:1789760793.465053:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.465073:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5865, 5865] 00000008:00010000:1.0:1789760793.465092:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.465098:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.465110:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24023040, 24027136) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.465123:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24023040=0x16e9000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.465184:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.465186:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24027136 00000020:00010000:1.0:1789760793.465195:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.465198:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.465201:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.465205:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.465208:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.465218:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.465225:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.465243:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d14516c0 00000008:00100000:1.0:1789760793.465342:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 24023040<>24027136 00000100:00100000:0.0:1789760793.465420:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388672:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.493118:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973185 gen 3 00000008:00010000:0.0:1789760793.493170:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d14516c0 00000020:00200000:0.0:1789760793.493178:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.493190:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.493211:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d14516c0 00000100:00100000:0.0:1789760793.493241:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388672:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.493427:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5865, 5865] 00000080:00200000:1.0:1789760793.493444:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.493451:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.493468:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24027136 i_blocks 47111 00000020:00200000:1.0:1789760793.493472:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.493482:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.493519:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24027136, count: 4096 00000080:00200000:1.0:1789760793.493526:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24027136, bytes: 4096 00000080:00200000:1.0:1789760793.493535:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.493544:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.493547:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24027136, 24031232) 0 0 00000080:00200000:1.0:1789760793.493551:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.493569:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5866, 5866] 00020000:00200000:1.0:1789760793.493577:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24027136 chunk: [24027136, 24031232) 24031232, 4096 00020000:00200000:1.0:1789760793.493582:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.493621:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5866, 5866] 00000008:00010000:1.0:1789760793.493638:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.493645:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.493658:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24027136, 24031232) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.493673:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24027136=0x16ea000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.493731:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.493734:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24031232 00000020:00010000:1.0:1789760793.493743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.493746:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.493749:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.493752:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.493755:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.493764:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.493770:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.493787:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451600 00000008:00100000:1.0:1789760793.493893:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 24027136<>24031232 00000100:00100000:0.0:1789760793.494476:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760793.494501:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388800:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.521619:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973186 gen 3 00000008:00010000:0.0:1789760793.521676:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451600 00000020:00200000:0.0:1789760793.521683:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.521694:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.521712:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451600 00000100:00100000:0.0:1789760793.521738:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388800:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.521840:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5866, 5866] 00000080:00200000:1.0:1789760793.521856:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.521862:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.521878:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24031232 i_blocks 47111 00000020:00200000:1.0:1789760793.521882:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.521891:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.521934:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24031232, count: 4096 00000080:00200000:1.0:1789760793.521941:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24031232, bytes: 4096 00000080:00200000:1.0:1789760793.521951:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.521960:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.521962:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24031232, 24035328) 0 0 00000080:00200000:1.0:1789760793.521967:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.521984:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5867, 5867] 00020000:00200000:1.0:1789760793.521992:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24031232 chunk: [24031232, 24035328) 24035328, 4096 00020000:00200000:1.0:1789760793.521997:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.522015:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5867, 5867] 00000008:00010000:1.0:1789760793.522033:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.522039:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.522058:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24031232, 24035328) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.522071:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24031232=0x16eb000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.522125:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.522128:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24035328 00000020:00010000:1.0:1789760793.522135:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.522138:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.522141:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.522144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.522147:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.522154:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.522160:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.522174:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760793.522274:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 24031232<>24035328 00000100:00100000:0.0:1789760793.522432:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388928:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.552859:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973187 gen 3 00000008:00010000:0.0:1789760793.552922:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:0.0:1789760793.552932:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.552942:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.552963:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:0.0:1789760793.552992:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507388928:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.553132:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5867, 5867] 00000080:00200000:1.0:1789760793.553151:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.553158:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.553172:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24035328 i_blocks 47111 00000020:00200000:1.0:1789760793.553176:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.553187:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.553222:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24035328, count: 4096 00000080:00200000:1.0:1789760793.553229:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24035328, bytes: 4096 00000080:00200000:1.0:1789760793.553239:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.553249:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.553251:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24035328, 24039424) 0 0 00000080:00200000:1.0:1789760793.553255:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.553270:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5868, 5868] 00020000:00200000:1.0:1789760793.553278:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24035328 chunk: [24035328, 24039424) 24039424, 4096 00020000:00200000:1.0:1789760793.553283:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.553300:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5868, 5868] 00000008:00010000:1.0:1789760793.553318:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.553324:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.553334:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24035328, 24039424) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.553347:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24035328=0x16ec000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.553821:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.553827:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24039424 00000020:00010000:1.0:1789760793.553836:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.553839:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.553842:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.553845:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.553848:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.553857:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.553866:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.553886:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760793.554033:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 24035328<>24039424 00000100:00100000:0.0:1789760793.554153:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507389056:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.591350:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973188 gen 3 00000008:00010000:0.0:1789760793.591446:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760793.591455:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.591467:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.591487:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760793.591519:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507389056:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.591827:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5868, 5868] 00000080:00200000:1.0:1789760793.591846:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.591853:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.591871:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24039424 i_blocks 47111 00000020:00200000:1.0:1789760793.591875:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.591885:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.591932:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24039424, count: 4096 00000080:00200000:1.0:1789760793.591940:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24039424, bytes: 4096 00000080:00200000:1.0:1789760793.591952:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.591964:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.591967:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24039424, 24043520) 0 0 00000080:00200000:1.0:1789760793.591971:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.591988:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5869, 5869] 00020000:00200000:1.0:1789760793.591998:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24039424 chunk: [24039424, 24043520) 24043520, 4096 00020000:00200000:1.0:1789760793.592003:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.592025:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5869, 5869] 00000008:00010000:1.0:1789760793.592150:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.592156:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.592170:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24039424, 24043520) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.592184:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24039424=0x16ed000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.592246:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.592248:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24043520 00000020:00010000:1.0:1789760793.592257:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.592260:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.592263:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.592267:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.592269:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.592278:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.592285:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.592305:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760793.592465:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 24039424<>24043520 00000100:00100000:0.0:1789760793.592642:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389184:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760793.651215:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760793.651231:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.651245:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.651270:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760793.651301:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389184:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.651449:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5869, 5869] 00000080:00200000:1.0:1789760793.651467:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.651473:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.651487:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24043520 i_blocks 47111 00000020:00200000:1.0:1789760793.651491:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.651500:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.651533:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24043520, count: 4096 00000080:00200000:1.0:1789760793.651540:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24043520, bytes: 4096 00000080:00200000:1.0:1789760793.651549:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.651558:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.651560:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24043520, 24047616) 0 0 00000080:00200000:1.0:1789760793.651565:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.651580:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5870, 5870] 00020000:00200000:1.0:1789760793.651588:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24043520 chunk: [24043520, 24047616) 24047616, 4096 00020000:00200000:1.0:1789760793.651593:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.651624:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5870, 5870] 00000008:00010000:1.0:1789760793.651643:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.651650:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.651660:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24043520, 24047616) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.651673:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24043520=0x16ee000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.651731:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.651734:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24047616 00000020:00010000:1.0:1789760793.651743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.651746:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.651748:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.651752:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.651754:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.651763:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.651769:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.651786:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760793.651891:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168700 - object 0x280000401:3 offset 24043520<>24047616 00000100:00100000:0.0:1789760793.651981:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760793.651999:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389312:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.696129:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973189 gen 3 00000100:00100000:0.0:1789760793.696175:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c8168700 x1876699507389312/t4294973190(4294973190) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760809 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:0.0:1789760793.696228:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760793.696235:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.696246:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.696337:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760793.696361:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389312:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.696571:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5870, 5870] 00000080:00200000:1.0:1789760793.696587:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.696593:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.696622:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24047616 i_blocks 47111 00000020:00200000:1.0:1789760793.696626:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.696636:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.696841:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24047616, count: 4096 00000080:00200000:1.0:1789760793.696850:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24047616, bytes: 4096 00000080:00200000:1.0:1789760793.696866:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.696874:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.696877:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24047616, 24051712) 0 0 00000080:00200000:1.0:1789760793.696881:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.696898:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5871, 5871] 00020000:00200000:1.0:1789760793.696907:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24047616 chunk: [24047616, 24051712) 24051712, 4096 00020000:00200000:1.0:1789760793.696914:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.696934:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5871, 5871] 00000008:00010000:1.0:1789760793.696953:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.696959:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.696970:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24047616, 24051712) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.696984:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24047616=0x16ef000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.697046:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.697049:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24051712 00000020:00010000:1.0:1789760793.697058:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.697061:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.697064:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.697068:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.697070:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.697083:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.697090:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.697109:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451840 00000008:00100000:1.0:1789760793.697219:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168e00 - object 0x280000401:3 offset 24047616<>24051712 00000100:00100000:0.0:1789760793.697292:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389440:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.738032:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973191 gen 3 00000008:00010000:0.0:1789760793.738100:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451840 00000020:00200000:0.0:1789760793.738109:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.738120:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.738243:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451840 00000100:00100000:0.0:1789760793.738283:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389440:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.738438:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5871, 5871] 00000080:00200000:1.0:1789760793.738456:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.738462:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.738478:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24051712 i_blocks 47111 00000020:00200000:1.0:1789760793.738482:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.738492:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.738525:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24051712, count: 4096 00000080:00200000:1.0:1789760793.738533:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24051712, bytes: 4096 00000080:00200000:1.0:1789760793.738542:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.738552:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.738554:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24051712, 24055808) 0 0 00000080:00200000:1.0:1789760793.738706:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.738723:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5872, 5872] 00020000:00200000:1.0:1789760793.738732:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24051712 chunk: [24051712, 24055808) 24055808, 4096 00020000:00200000:1.0:1789760793.738737:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.738756:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5872, 5872] 00000008:00010000:1.0:1789760793.738772:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.738777:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.738788:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24051712, 24055808) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.738801:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24051712=0x16f0000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.738858:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.738861:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24055808 00000020:00010000:1.0:1789760793.738868:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.738871:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.738875:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.738879:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.738882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.738889:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.738896:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.738913:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451c00 00000008:00100000:1.0:1789760793.739017:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169f80 - object 0x280000401:3 offset 24051712<>24055808 00000100:00100000:0.0:1789760793.739083:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507389568:192.168.203.141@tcp:4:dd.0 00000008:00010000:0.0:1789760793.769343:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451c00 00000020:00200000:0.0:1789760793.769360:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.769372:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.769439:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451c00 00000100:00100000:0.0:1789760793.769475:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507389568:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.769671:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5872, 5872] 00000080:00200000:1.0:1789760793.769687:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.769694:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.769709:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24055808 i_blocks 47111 00000020:00200000:1.0:1789760793.769713:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.769723:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.769933:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24055808, count: 4096 00000080:00200000:1.0:1789760793.769941:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24055808, bytes: 4096 00000080:00200000:1.0:1789760793.769951:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.769961:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.769963:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24055808, 24059904) 0 0 00000080:00200000:1.0:1789760793.769968:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.769985:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5873, 5873] 00020000:00200000:1.0:1789760793.769993:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24055808 chunk: [24055808, 24059904) 24059904, 4096 00020000:00200000:1.0:1789760793.769998:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.770017:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5873, 5873] 00000008:00010000:1.0:1789760793.770036:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.770041:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.770053:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24055808, 24059904) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.770067:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24055808=0x16f1000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.770126:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.770129:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24059904 00000020:00010000:1.0:1789760793.770137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.770140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.770143:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.770147:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.770150:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.770158:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.770165:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.770182:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451b40 00000008:00100000:1.0:1789760793.770286:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816a680 - object 0x280000401:3 offset 24055808<>24059904 00000100:00100000:0.0:1789760793.770358:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389696:192.168.203.141@tcp:4:dd.0 00000100:00100000:0.0:1789760793.803653:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973193 gen 3 00000008:00010000:0.0:1789760793.803723:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451b40 00000020:00200000:0.0:1789760793.803731:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:0.0:1789760793.803741:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:0.0:1789760793.803801:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451b40 00000100:00100000:0.0:1789760793.803830:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389696:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.806472:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5873, 5873] 00000080:00200000:1.0:1789760793.806504:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.806512:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.806605:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24059904 i_blocks 47111 00000020:00200000:1.0:1789760793.806611:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.806628:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.806665:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24059904, count: 4096 00000080:00200000:1.0:1789760793.806684:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24059904, bytes: 4096 00000080:00200000:1.0:1789760793.806701:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.806712:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.806715:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24059904, 24064000) 0 0 00000080:00200000:1.0:1789760793.806719:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.806736:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5874, 5874] 00020000:00200000:1.0:1789760793.806745:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24059904 chunk: [24059904, 24064000) 24064000, 4096 00020000:00200000:1.0:1789760793.806750:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.806774:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5874, 5874] 00000008:00010000:1.0:1789760793.806794:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.806800:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.806813:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24059904, 24064000) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.806827:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24059904=0x16f2000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.806915:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.806918:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24064000 00000020:00010000:1.0:1789760793.806935:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.806938:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.806941:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.806945:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.806948:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.806958:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.806965:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.806999:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760793.807621:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 24059904<>24064000 00000100:00100000:0.0:1789760793.807953:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760793.807982:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389824:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760793.856012:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973194 gen 3 00000008:00010000:1.0:1789760793.856127:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760793.856136:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760793.856148:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760793.856170:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760793.856203:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507389824:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760793.856322:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5874, 5874] 00000080:00200000:2.0:1789760793.856338:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760793.856344:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760793.856359:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24064000 i_blocks 47111 00000020:00200000:2.0:1789760793.856362:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760793.856373:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760793.856430:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24064000, count: 4096 00000080:00200000:2.0:1789760793.856441:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24064000, bytes: 4096 00000080:00200000:2.0:1789760793.856451:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760793.856460:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760793.856462:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24064000, 24068096) 0 0 00000080:00200000:2.0:1789760793.856467:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760793.856482:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5875, 5875] 00020000:00200000:2.0:1789760793.856491:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24064000 chunk: [24064000, 24068096) 24068096, 4096 00020000:00200000:2.0:1789760793.856494:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760793.856513:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5875, 5875] 00000008:00010000:2.0:1789760793.856530:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760793.856536:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760793.856549:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24064000, 24068096) append 0 append lockpos 0 00000080:00200000:2.0:1789760793.856562:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24064000=0x16f3000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760793.856838:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760793.856842:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24068096 00000020:00010000:2.0:1789760793.856852:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760793.856856:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.856859:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.856862:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760793.856866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760793.856874:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760793.856880:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760793.856901:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760793.857005:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 24064000<>24068096 00000100:00100000:2.0:1789760793.894282:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973195 gen 3 00000008:00010000:2.0:1789760793.894334:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760793.894343:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760793.894353:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760793.894419:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760793.894449:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507389952:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.894568:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5875, 5875] 00000080:00200000:1.0:1789760793.894584:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.894590:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.894617:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24068096 i_blocks 47111 00000020:00200000:1.0:1789760793.894621:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.894642:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.894679:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24068096, count: 4096 00000080:00200000:1.0:1789760793.894686:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24068096, bytes: 4096 00000080:00200000:1.0:1789760793.894698:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.894711:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.894714:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24068096, 24072192) 0 0 00000080:00200000:1.0:1789760793.894719:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.894736:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5876, 5876] 00020000:00200000:1.0:1789760793.894746:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24068096 chunk: [24068096, 24072192) 24072192, 4096 00020000:00200000:1.0:1789760793.894751:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.894769:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5876, 5876] 00000008:00010000:1.0:1789760793.894787:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.894793:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.894803:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24068096, 24072192) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.894818:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24068096=0x16f4000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.894879:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.894882:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24072192 00000020:00010000:1.0:1789760793.894891:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.894894:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.894909:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.894912:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.894915:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.894922:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.894929:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.894945:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760793.895059:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 24068096<>24072192 00000100:00100000:0.0:1789760793.895228:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507390080:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760793.926182:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973196 gen 3 00000008:00010000:1.0:1789760793.926233:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760793.926242:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760793.926254:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760793.926323:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760793.926354:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507390080:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760793.926546:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5876, 5876] 00000080:00200000:2.0:1789760793.926564:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760793.926570:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760793.926612:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24072192 i_blocks 47111 00000020:00200000:2.0:1789760793.926616:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760793.926627:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760793.926726:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24072192, count: 4096 00000080:00200000:2.0:1789760793.926733:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24072192, bytes: 4096 00000080:00200000:2.0:1789760793.926744:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760793.926754:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760793.926756:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24072192, 24076288) 0 0 00000080:00200000:2.0:1789760793.926761:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760793.926779:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5877, 5877] 00020000:00200000:2.0:1789760793.926788:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24072192 chunk: [24072192, 24076288) 24076288, 4096 00020000:00200000:2.0:1789760793.926793:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760793.926814:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5877, 5877] 00000008:00010000:2.0:1789760793.926835:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760793.926841:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760793.926853:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24072192, 24076288) append 0 append lockpos 0 00000080:00200000:2.0:1789760793.926869:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24072192=0x16f5000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760793.926991:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760793.926994:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24076288 00000020:00010000:2.0:1789760793.927004:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760793.927008:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.927011:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.927014:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760793.927017:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760793.927024:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760793.927030:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760793.927044:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760793.927154:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 24072192<>24076288 00000100:00100000:2.0:1789760793.927201:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760793.927220:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507390208:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760793.949329:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973197 gen 3 00000008:00010000:2.0:1789760793.949419:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760793.949428:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760793.949439:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760793.949469:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760793.949503:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507390208:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760793.949625:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5877, 5877] 00000080:00200000:1.0:1789760793.949705:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760793.949711:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760793.949726:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24076288 i_blocks 47111 00000020:00200000:1.0:1789760793.949730:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760793.949740:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760793.949773:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24076288, count: 4096 00000080:00200000:1.0:1789760793.949780:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24076288, bytes: 4096 00000080:00200000:1.0:1789760793.949790:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760793.949798:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760793.949801:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24076288, 24080384) 0 0 00000080:00200000:1.0:1789760793.949805:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760793.949820:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5878, 5878] 00020000:00200000:1.0:1789760793.949829:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24076288 chunk: [24076288, 24080384) 24080384, 4096 00020000:00200000:1.0:1789760793.949833:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760793.949853:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5878, 5878] 00000008:00010000:1.0:1789760793.949871:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760793.949878:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760793.949891:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24076288, 24080384) append 0 append lockpos 0 00000080:00200000:1.0:1789760793.949907:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24076288=0x16f6000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760793.949969:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760793.949972:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24080384 00000020:00010000:1.0:1789760793.949982:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:1.0:1789760793.949987:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.949990:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760793.949993:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760793.949996:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760793.950004:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760793.950011:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760793.950028:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760793.950133:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 24076288<>24080384 00000100:00100000:0.0:1789760793.950204:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507390336:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760793.970988:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973198 gen 3 00000008:00010000:1.0:1789760793.971053:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760793.971061:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760793.971073:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760793.971161:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760793.971198:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507390336:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760793.971802:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5878, 5878] 00000080:00200000:2.0:1789760793.971821:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760793.971827:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760793.971846:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24080384 i_blocks 47111 00000020:00200000:2.0:1789760793.971851:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760793.971867:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760793.971915:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24080384, count: 4096 00000080:00200000:2.0:1789760793.971923:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24080384, bytes: 4096 00000080:00200000:2.0:1789760793.971935:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760793.971953:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760793.972085:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24080384, 24084480) 0 0 00000080:00200000:2.0:1789760793.972091:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760793.972175:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5879, 5879] 00020000:00200000:2.0:1789760793.972186:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24080384 chunk: [24080384, 24084480) 24084480, 4096 00020000:00200000:2.0:1789760793.972193:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760793.972217:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5879, 5879] 00000008:00010000:2.0:1789760793.972242:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760793.972249:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760793.972264:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24080384, 24084480) append 0 append lockpos 0 00000080:00200000:2.0:1789760793.972283:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24080384=0x16f7000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760793.972360:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760793.972362:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24084480 00000020:00010000:2.0:1789760793.972370:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_release():245 00000020:00010000:2.0:1789760793.972373:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.972375:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038ab8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760793.972406:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be95d8 (ffff8d25c8038488/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760793.972409:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038488/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760793.972418:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760793.972427:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760793.972455:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760793.972579:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 24080384<>24084480 00000100:00100000:2.0:1789760793.972656:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507390464:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.013540:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973199 gen 3 00000008:00010000:2.0:1789760794.013600:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760794.013608:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.013619:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.013639:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760794.013669:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507390464:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.015307:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390592:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.054279:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973200 gen 3 00000008:00010000:2.0:1789760794.054317:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c780 00000020:00200000:2.0:1789760794.054323:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.054331:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.054349:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c780 00000100:00100000:2.0:1789760794.054370:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390592:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.057048:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760794.057427:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390720:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.108804:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973201 gen 3 00000008:00010000:2.0:1789760794.108838:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5ca80 00000020:00200000:2.0:1789760794.108846:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.108855:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.108940:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5ca80 00000100:00100000:2.0:1789760794.108960:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390720:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.115637:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390848:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.154685:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973202 gen 3 00000008:00010000:2.0:1789760794.154731:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000020:00200000:2.0:1789760794.154737:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.154746:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.154763:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000100:00100000:2.0:1789760794.154786:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390848:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.155505:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760794.155518:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390976:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.198446:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973203 gen 3 00000008:00010000:2.0:1789760794.198486:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c900 00000020:00200000:2.0:1789760794.198492:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.198500:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.198575:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c900 00000100:00100000:2.0:1789760794.198608:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507390976:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.201281:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391104:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.231493:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973204 gen 3 00000008:00010000:2.0:1789760794.231528:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c900 00000020:00200000:2.0:1789760794.231534:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.231542:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.231623:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c900 00000100:00100000:2.0:1789760794.231648:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391104:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760794.231984:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24104960 i_blocks 47111 00000020:00200000:2.0:1789760794.231988:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760794.231996:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760794.232022:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24104960, count: 4096 00000080:00200000:2.0:1789760794.232155:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24104960, bytes: 4096 00000080:00200000:2.0:1789760794.232167:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760794.232174:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760794.232176:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24104960, 24109056) 0 0 00000080:00200000:2.0:1789760794.232180:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760794.232193:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5885, 5885] 00020000:00200000:2.0:1789760794.232200:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24104960 chunk: [24104960, 24109056) 24109056, 4096 00020000:00200000:2.0:1789760794.232204:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760794.232219:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5885, 5885] 00000008:00010000:2.0:1789760794.232233:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760794.232238:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760794.232247:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24104960, 24109056) append 0 append lockpos 0 00000080:00200000:2.0:1789760794.232259:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24104960=0x16fd000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760794.232300:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760794.232303:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24109056 00000020:00010000:2.0:1789760794.232309:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be9d58 (ffff8d25c80383f8/0) at cl_lock_release():245 00000020:00010000:2.0:1789760794.232312:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be9d58 (ffff8d25c80383f8/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.232315:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.232318:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be9d58 (ffff8d25c80383f8/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760794.232321:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760794.232327:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760794.232332:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760794.232371:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760794.232524:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 24104960<>24109056 00000100:00100000:2.0:1789760794.232552:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760794.232564:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391232:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.260682:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973205 gen 3 00000008:00010000:2.0:1789760794.260717:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50aca80 00000020:00200000:2.0:1789760794.260723:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.260731:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.260751:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50aca80 00000100:00100000:2.0:1789760794.260776:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391232:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.261462:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391360:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.296127:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973206 gen 3 00000008:00010000:2.0:1789760794.296165:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000020:00200000:2.0:1789760794.296171:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.296179:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.296262:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c6c0 00000100:00100000:2.0:1789760794.296289:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391360:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.300143:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507391488:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.346753:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973207 gen 3 00000008:00010000:2.0:1789760794.346804:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5ca80 00000020:00200000:2.0:1789760794.346810:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.346820:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.346841:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5ca80 00000100:00100000:2.0:1789760794.346863:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507391488:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.347716:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391616:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.387076:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973208 gen 3 00000008:00010000:2.0:1789760794.387108:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c780 00000020:00200000:2.0:1789760794.387114:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.387123:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.387140:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c780 00000100:00100000:2.0:1789760794.387161:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391616:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.387780:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760794.387792:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391744:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760794.419021:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5cd80 00000020:00200000:2.0:1789760794.419041:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.419051:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.419074:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5cd80 00000100:00100000:2.0:1789760794.419100:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507391744:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.425700:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760794.425731:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507391872:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.467255:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973210 gen 3 00000008:00010000:2.0:1789760794.467298:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c300 00000020:00200000:2.0:1789760794.467304:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.467311:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.467330:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c300 00000100:00100000:2.0:1789760794.467358:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507391872:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.468363:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760794.468397:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507392000:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760794.511732:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c000 00000020:00200000:2.0:1789760794.511746:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.511766:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.511788:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c000 00000100:00100000:2.0:1789760794.511812:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507392000:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.513304:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507392128:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.559247:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973212 gen 3 00000008:00010000:2.0:1789760794.559291:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c180 00000020:00200000:2.0:1789760794.559297:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.559305:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.559692:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c180 00000100:00100000:2.0:1789760794.559716:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507392128:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.562413:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507392256:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.600698:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973213 gen 3 00000008:00010000:2.0:1789760794.600736:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5cf00 00000020:00200000:2.0:1789760794.600742:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.600751:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.600774:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5cf00 00000100:00100000:2.0:1789760794.600798:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507392256:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.602561:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507392384:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760794.647198:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973214 gen 3 00000008:00010000:2.0:1789760794.647235:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c3c0 00000020:00200000:2.0:1789760794.647243:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760794.647251:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760794.647271:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c3c0 00000100:00100000:2.0:1789760794.647296:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507392384:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760794.649025:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5894, 5894] 00000080:00200000:1.0:1789760794.649053:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760794.649060:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760794.649090:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24145920 i_blocks 49159 00000020:00200000:1.0:1789760794.649094:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760794.649116:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760794.649152:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24145920, count: 4096 00000080:00200000:1.0:1789760794.649161:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24145920, bytes: 4096 00000080:00200000:1.0:1789760794.649175:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760794.649185:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760794.649187:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24145920, 24150016) 0 0 00000080:00200000:1.0:1789760794.649192:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760794.649208:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5895, 5895] 00020000:00200000:1.0:1789760794.649218:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24145920 chunk: [24145920, 24150016) 24150016, 4096 00020000:00200000:1.0:1789760794.649222:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760794.649245:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5895, 5895] 00000008:00010000:1.0:1789760794.649265:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760794.649272:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760794.649284:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24145920, 24150016) append 0 append lockpos 0 00000080:00200000:1.0:1789760794.649299:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24145920=0x1707000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760794.649365:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760794.649368:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24150016 00000020:00010000:1.0:1789760794.649431:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760794.649436:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760794.649440:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760794.649443:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760794.649446:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760794.649456:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760794.649464:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760794.649489:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760794.649844:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 24145920<>24150016 00000100:00100000:1.0:1789760794.649903:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:1.0:1789760794.649928:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507392512:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760794.716090:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973215 gen 3 00000008:00010000:1.0:1789760794.716138:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760794.716144:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760794.716152:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760794.716225:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760794.716251:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507392512:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760794.717963:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5895, 5895] 00000080:00200000:2.0:1789760794.717980:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760794.717985:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760794.717998:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24150016 i_blocks 49159 00000020:00200000:2.0:1789760794.718002:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760794.718012:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760794.718037:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24150016, count: 4096 00000080:00200000:2.0:1789760794.718043:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24150016, bytes: 4096 00000080:00200000:2.0:1789760794.718053:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760794.718061:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760794.718062:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24150016, 24154112) 0 0 00000080:00200000:2.0:1789760794.718066:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760794.718078:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5896, 5896] 00020000:00200000:2.0:1789760794.718083:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24150016 chunk: [24150016, 24154112) 24154112, 4096 00020000:00200000:2.0:1789760794.718087:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760794.718103:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5896, 5896] 00000008:00010000:2.0:1789760794.718221:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760794.718227:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760794.718239:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24150016, 24154112) append 0 append lockpos 0 00000080:00200000:2.0:1789760794.718250:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24150016=0x1708000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760794.718300:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760794.718302:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24154112 00000020:00010000:2.0:1789760794.718308:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760794.718311:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.718471:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.718476:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760794.718480:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760794.718488:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760794.718494:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760794.718517:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760794.718759:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022a00 - object 0x280000401:3 offset 24150016<>24154112 00000080:00200000:2.0:1789760794.780500:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5896, 5896] 00000080:00200000:2.0:1789760794.780519:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760794.780526:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760794.780628:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24154112 i_blocks 49159 00000020:00200000:2.0:1789760794.780633:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760794.780643:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760794.780679:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24154112, count: 4096 00000080:00200000:2.0:1789760794.780687:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24154112, bytes: 4096 00000080:00200000:2.0:1789760794.780699:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760794.780708:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760794.780710:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24154112, 24158208) 0 0 00000080:00200000:2.0:1789760794.780715:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760794.780732:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5897, 5897] 00020000:00200000:2.0:1789760794.780741:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24154112 chunk: [24154112, 24158208) 24158208, 4096 00020000:00200000:2.0:1789760794.780745:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760794.780765:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5897, 5897] 00000008:00010000:2.0:1789760794.780784:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760794.780790:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760794.780802:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24154112, 24158208) append 0 append lockpos 0 00000080:00200000:2.0:1789760794.780817:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24154112=0x1709000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760794.780883:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760794.780886:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24158208 00000020:00010000:2.0:1789760794.781184:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760794.781190:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.781193:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.781197:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760794.781200:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760794.781210:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760794.781219:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760794.781246:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760794.781360:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022d80 - object 0x280000401:3 offset 24154112<>24158208 00000080:00200000:2.0:1789760794.851279:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5897, 5897] 00000080:00200000:2.0:1789760794.851304:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760794.851311:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760794.851335:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24158208 i_blocks 49159 00000020:00200000:2.0:1789760794.851341:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760794.851354:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760794.851429:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24158208, count: 4096 00000080:00200000:2.0:1789760794.851437:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24158208, bytes: 4096 00000080:00200000:2.0:1789760794.851447:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760794.851455:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760794.851457:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24158208, 24162304) 0 0 00000080:00200000:2.0:1789760794.851461:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760794.851473:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5898, 5898] 00020000:00200000:2.0:1789760794.851480:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24158208 chunk: [24158208, 24162304) 24162304, 4096 00020000:00200000:2.0:1789760794.851483:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760794.851500:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5898, 5898] 00000008:00010000:2.0:1789760794.851515:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760794.851520:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760794.851529:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24158208, 24162304) append 0 append lockpos 0 00000080:00200000:2.0:1789760794.851540:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24158208=0x170a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760794.851716:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760794.851719:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24162304 00000020:00010000:2.0:1789760794.851727:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760794.851730:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.851733:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.851736:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760794.851740:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760794.851747:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760794.851753:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760794.851771:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760794.851923:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021180 - object 0x280000401:3 offset 24158208<>24162304 00000080:00200000:2.0:1789760794.895075:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5898, 5898] 00000080:00200000:2.0:1789760794.895096:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760794.895103:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760794.895117:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24162304 i_blocks 49159 00000020:00200000:2.0:1789760794.895122:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760794.895134:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760794.895159:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24162304, count: 4096 00000080:00200000:2.0:1789760794.895165:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24162304, bytes: 4096 00000080:00200000:2.0:1789760794.895175:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760794.895183:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760794.895185:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24162304, 24166400) 0 0 00000080:00200000:2.0:1789760794.895189:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760794.895200:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5899, 5899] 00020000:00200000:2.0:1789760794.895207:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24162304 chunk: [24162304, 24166400) 24166400, 4096 00020000:00200000:2.0:1789760794.895210:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760794.895227:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5899, 5899] 00000008:00010000:2.0:1789760794.895240:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760794.895245:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760794.895253:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24162304, 24166400) append 0 append lockpos 0 00000080:00200000:2.0:1789760794.895264:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24162304=0x170b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760794.895313:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760794.895316:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24166400 00000020:00010000:2.0:1789760794.895322:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760794.895326:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.895328:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.895331:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760794.895334:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760794.895341:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760794.895346:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760794.895365:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760794.895499:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020700 - object 0x280000401:3 offset 24162304<>24166400 00000080:00200000:2.0:1789760794.989139:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5899, 5899] 00000080:00200000:2.0:1789760794.989225:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760794.989232:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760794.989250:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24166400 i_blocks 49159 00000020:00200000:2.0:1789760794.989255:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760794.989266:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760794.989301:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24166400, count: 4096 00000080:00200000:2.0:1789760794.989416:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24166400, bytes: 4096 00000080:00200000:2.0:1789760794.989432:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760794.989444:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760794.989446:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24166400, 24170496) 0 0 00000080:00200000:2.0:1789760794.989451:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760794.989468:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5900, 5900] 00020000:00200000:2.0:1789760794.989476:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24166400 chunk: [24166400, 24170496) 24170496, 4096 00020000:00200000:2.0:1789760794.989481:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760794.989503:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5900, 5900] 00000008:00010000:2.0:1789760794.989523:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760794.989529:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760794.989541:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24166400, 24170496) append 0 append lockpos 0 00000080:00200000:2.0:1789760794.989557:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24166400=0x170c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760794.989629:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760794.989632:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24170496 00000020:00010000:2.0:1789760794.989640:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760794.989644:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.989647:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760794.989651:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760794.989655:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760794.989663:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760794.989670:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.059519:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5900, 5900] 00000080:00200000:2.0:1789760795.059533:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.059538:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.059551:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24170496 i_blocks 49159 00000020:00200000:2.0:1789760795.059554:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.059562:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.059653:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24170496, count: 4096 00000080:00200000:2.0:1789760795.059659:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24170496, bytes: 4096 00000080:00200000:2.0:1789760795.059666:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.059673:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.059675:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24170496, 24174592) 0 0 00000080:00200000:2.0:1789760795.059679:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.059692:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5901, 5901] 00020000:00200000:2.0:1789760795.059699:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24170496 chunk: [24170496, 24174592) 24174592, 4096 00020000:00200000:2.0:1789760795.059703:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.059717:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5901, 5901] 00000008:00010000:2.0:1789760795.059729:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.059734:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.059742:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24170496, 24174592) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.059753:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24170496=0x170d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.059793:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.059795:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24174592 00000020:00010000:2.0:1789760795.059802:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.059805:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.059808:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.059811:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.059814:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.059820:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.059827:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.059840:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760795.059916:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020000 - object 0x280000401:3 offset 24170496<>24174592 00000008:00010000:2.0:1789760795.170598:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac900 00000020:00200000:2.0:1789760795.170608:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760795.170617:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760795.170743:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac900 00000100:00100000:2.0:1789760795.170765:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507393280:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760795.170822:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5901, 5901] 00000080:00200000:1.0:1789760795.170835:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760795.170840:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760795.170851:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24174592 i_blocks 49159 00000020:00200000:1.0:1789760795.170854:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760795.170862:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760795.170886:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24174592, count: 4096 00000080:00200000:1.0:1789760795.170891:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24174592, bytes: 4096 00000080:00200000:1.0:1789760795.170947:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760795.170955:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760795.170957:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24174592, 24178688) 0 0 00000080:00200000:1.0:1789760795.170961:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760795.170973:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5902, 5902] 00020000:00200000:1.0:1789760795.170979:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24174592 chunk: [24174592, 24178688) 24178688, 4096 00020000:00200000:1.0:1789760795.170982:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760795.170997:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5902, 5902] 00000008:00010000:1.0:1789760795.171009:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760795.171014:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760795.171022:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24174592, 24178688) append 0 append lockpos 0 00000080:00200000:1.0:1789760795.171140:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24174592=0x170e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760795.171184:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760795.171187:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24178688 00000020:00010000:1.0:1789760795.171193:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760795.171196:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760795.171199:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760795.171202:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760795.171205:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760795.171212:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760795.171217:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760795.171230:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760795.171305:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c816aa00 - object 0x280000401:3 offset 24174592<>24178688 00000100:00100000:0.0:1789760795.171413:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:0.0:1789760795.171430:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507393408:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760795.227294:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973221 gen 3 00000100:00100000:1.0:1789760795.227323:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25c816aa00 x1876699507393408/t4294973222(4294973222) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760811 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760795.227360:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760795.227366:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760795.227375:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760795.227469:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760795.227490:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507393408:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760795.227545:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5902, 5902] 00000080:00200000:2.0:1789760795.227558:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.227564:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.227575:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24178688 i_blocks 49159 00000020:00200000:2.0:1789760795.227579:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.227597:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.227692:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24178688, count: 4096 00000080:00200000:2.0:1789760795.227699:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24178688, bytes: 4096 00000080:00200000:2.0:1789760795.227707:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.227715:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.227717:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24178688, 24182784) 0 0 00000080:00200000:2.0:1789760795.227722:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.227736:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5903, 5903] 00020000:00200000:2.0:1789760795.227743:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24178688 chunk: [24178688, 24182784) 24182784, 4096 00020000:00200000:2.0:1789760795.227748:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.227764:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5903, 5903] 00000008:00010000:2.0:1789760795.227776:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.227781:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.227790:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24178688, 24182784) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.227800:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24178688=0x170f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.227844:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.227846:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24182784 00000020:00010000:2.0:1789760795.227853:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.227857:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.227860:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.227863:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.227866:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.227873:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.227879:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.227893:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760795.228044:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023100 - object 0x280000401:3 offset 24178688<>24182784 00000080:00200000:2.0:1789760795.303662:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5903, 5903] 00000080:00200000:2.0:1789760795.303675:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.303680:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.303692:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24182784 i_blocks 49159 00000020:00200000:2.0:1789760795.303695:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.303703:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.303729:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24182784, count: 4096 00000080:00200000:2.0:1789760795.303736:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24182784, bytes: 4096 00000080:00200000:2.0:1789760795.303743:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.303750:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.303752:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24182784, 24186880) 0 0 00000080:00200000:2.0:1789760795.303755:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.303768:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5904, 5904] 00020000:00200000:2.0:1789760795.303775:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24182784 chunk: [24182784, 24186880) 24186880, 4096 00020000:00200000:2.0:1789760795.303778:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.303792:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5904, 5904] 00000008:00010000:2.0:1789760795.303805:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.303810:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.303818:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24182784, 24186880) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.303828:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24182784=0x1710000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.303873:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.303875:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24186880 00000020:00010000:2.0:1789760795.303882:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.303886:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.303889:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.303892:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.303895:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.303901:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.303907:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.303921:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760795.303999:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023b80 - object 0x280000401:3 offset 24182784<>24186880 00000080:00200000:1.0:1789760795.341706:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5904, 5904] 00000080:00200000:1.0:1789760795.341719:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760795.341724:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760795.341736:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24186880 i_blocks 49159 00000020:00200000:1.0:1789760795.341738:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760795.341746:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760795.341773:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24186880, count: 4096 00000080:00200000:1.0:1789760795.341778:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24186880, bytes: 4096 00000080:00200000:1.0:1789760795.341786:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760795.341794:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760795.341796:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24186880, 24190976) 0 0 00000080:00200000:1.0:1789760795.341800:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760795.341812:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5905, 5905] 00020000:00200000:1.0:1789760795.341819:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24186880 chunk: [24186880, 24190976) 24190976, 4096 00020000:00200000:1.0:1789760795.341823:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760795.341838:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5905, 5905] 00000008:00010000:1.0:1789760795.341854:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760795.341860:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760795.341869:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24186880, 24190976) append 0 append lockpos 0 00000080:00200000:1.0:1789760795.341880:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24186880=0x1711000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760795.341926:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760795.341929:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24190976 00000020:00010000:1.0:1789760795.341936:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760795.341940:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760795.341942:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760795.341946:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760795.341949:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760795.341956:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760795.341963:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760795.341978:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760795.342079:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 24186880<>24190976 00000100:00100000:0.0:1789760795.342152:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507393792:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760795.391744:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973225 gen 3 00000008:00010000:1.0:1789760795.391784:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760795.391791:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760795.391800:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760795.391819:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760795.391842:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507393792:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760795.442556:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5906, 5906] 00000080:00200000:2.0:1789760795.442568:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.442573:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.442631:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24195072 i_blocks 49159 00000020:00200000:2.0:1789760795.442635:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.442644:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.442670:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24195072, count: 4096 00000080:00200000:2.0:1789760795.442676:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24195072, bytes: 4096 00000080:00200000:2.0:1789760795.442684:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.442690:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.442693:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24195072, 24199168) 0 0 00000080:00200000:2.0:1789760795.442696:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.442710:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5907, 5907] 00020000:00200000:2.0:1789760795.442717:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24195072 chunk: [24195072, 24199168) 24199168, 4096 00020000:00200000:2.0:1789760795.442721:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.442736:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5907, 5907] 00000008:00010000:2.0:1789760795.442750:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.442755:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.442764:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24195072, 24199168) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.442774:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24195072=0x1713000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.442816:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.442819:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24199168 00000020:00010000:2.0:1789760795.442825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.442829:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.442832:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.442836:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.442838:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.442845:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.442851:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.442870:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760795.443129:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 24195072<>24199168 00010000:00010000:0.0:1789760795.451608:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:2.0:1789760795.498299:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5907, 5907] 00000080:00200000:2.0:1789760795.498311:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.498316:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.498328:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24199168 i_blocks 49159 00000020:00200000:2.0:1789760795.498333:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.498341:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.498370:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24199168, count: 4096 00000080:00200000:2.0:1789760795.498411:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24199168, bytes: 4096 00000080:00200000:2.0:1789760795.498419:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.498426:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.498428:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24199168, 24203264) 0 0 00000080:00200000:2.0:1789760795.498432:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.498444:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5908, 5908] 00020000:00200000:2.0:1789760795.498450:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24199168 chunk: [24199168, 24203264) 24203264, 4096 00020000:00200000:2.0:1789760795.498454:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.498468:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5908, 5908] 00000008:00010000:2.0:1789760795.498482:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.498487:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.498495:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24199168, 24203264) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.498505:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24199168=0x1714000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.498542:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.498544:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24203264 00000020:00010000:2.0:1789760795.498550:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.498553:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.498556:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.498559:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.498562:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.498567:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.498572:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.498595:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760795.498755:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 24199168<>24203264 00000080:00200000:2.0:1789760795.569812:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5908, 5908] 00000080:00200000:2.0:1789760795.569823:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.569829:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.569840:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24203264 i_blocks 49159 00000020:00200000:2.0:1789760795.569843:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.569850:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.569873:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24203264, count: 4096 00000080:00200000:2.0:1789760795.569878:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24203264, bytes: 4096 00000080:00200000:2.0:1789760795.569885:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.569892:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.569894:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24203264, 24207360) 0 0 00000080:00200000:2.0:1789760795.569907:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.569919:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5909, 5909] 00020000:00200000:2.0:1789760795.569925:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24203264 chunk: [24203264, 24207360) 24207360, 4096 00020000:00200000:2.0:1789760795.569928:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.569941:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5909, 5909] 00000008:00010000:2.0:1789760795.569953:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.569957:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.569965:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24203264, 24207360) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.569975:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24203264=0x1715000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.570013:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.570015:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24207360 00000020:00010000:2.0:1789760795.570020:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.570292:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.570298:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.570301:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.570304:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.570310:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.570316:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.570329:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760795.570430:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 24203264<>24207360 00000080:00200000:2.0:1789760795.607745:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5909, 5909] 00000080:00200000:2.0:1789760795.607758:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.607762:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.607773:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24207360 i_blocks 49159 00000020:00200000:2.0:1789760795.607776:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.607783:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.607806:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24207360, count: 4096 00000080:00200000:2.0:1789760795.607811:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24207360, bytes: 4096 00000080:00200000:2.0:1789760795.607818:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.607824:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.607826:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24207360, 24211456) 0 0 00000080:00200000:2.0:1789760795.607830:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.607841:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5910, 5910] 00020000:00200000:2.0:1789760795.607847:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24207360 chunk: [24207360, 24211456) 24211456, 4096 00020000:00200000:2.0:1789760795.607851:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.607864:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5910, 5910] 00000008:00010000:2.0:1789760795.607877:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.607881:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.607889:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24207360, 24211456) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.607898:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24207360=0x1716000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.607940:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.607942:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24211456 00000020:00010000:2.0:1789760795.607949:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.607953:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.607956:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.607959:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.607962:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.607968:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.607974:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.607986:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760795.608067:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 24207360<>24211456 00000080:00200000:2.0:1789760795.654845:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5910, 5910] 00000080:00200000:2.0:1789760795.654858:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.654863:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.654876:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24211456 i_blocks 49159 00000020:00200000:2.0:1789760795.654880:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.654901:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.654927:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24211456, count: 4096 00000080:00200000:2.0:1789760795.654933:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24211456, bytes: 4096 00000080:00200000:2.0:1789760795.654941:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.654948:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.654951:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24211456, 24215552) 0 0 00000080:00200000:2.0:1789760795.654955:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.654967:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5911, 5911] 00020000:00200000:2.0:1789760795.654975:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24211456 chunk: [24211456, 24215552) 24215552, 4096 00020000:00200000:2.0:1789760795.654978:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.654994:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5911, 5911] 00000008:00010000:2.0:1789760795.655007:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.655011:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.655020:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24211456, 24215552) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.655038:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24211456=0x1717000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.655086:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.655089:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24215552 00000020:00010000:2.0:1789760795.655096:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.655100:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.655102:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.655106:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.655108:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.655115:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.655121:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.655135:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760795.655211:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 24211456<>24215552 00000080:00200000:2.0:1789760795.694120:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5911, 5911] 00000080:00200000:2.0:1789760795.694133:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.694139:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.694152:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24215552 i_blocks 49159 00000020:00200000:2.0:1789760795.694155:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.694163:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.694187:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24215552, count: 4096 00000080:00200000:2.0:1789760795.694193:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24215552, bytes: 4096 00000080:00200000:2.0:1789760795.694201:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.694208:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.694211:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24215552, 24219648) 0 0 00000080:00200000:2.0:1789760795.694214:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.694226:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5912, 5912] 00020000:00200000:2.0:1789760795.694233:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24215552 chunk: [24215552, 24219648) 24219648, 4096 00020000:00200000:2.0:1789760795.694237:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.694251:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5912, 5912] 00000008:00010000:2.0:1789760795.694265:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.694269:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.694278:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24215552, 24219648) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.694289:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24215552=0x1718000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.694331:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.694334:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24219648 00000020:00010000:2.0:1789760795.694341:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.694344:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.694347:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.694351:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.694354:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.694360:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.694366:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.694425:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760795.694503:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 24215552<>24219648 00000080:00200000:2.0:1789760795.729321:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5912, 5912] 00000080:00200000:2.0:1789760795.729336:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.729341:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.729354:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24219648 i_blocks 49159 00000020:00200000:2.0:1789760795.729357:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.729365:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.729415:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24219648, count: 4096 00000080:00200000:2.0:1789760795.729422:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24219648, bytes: 4096 00000080:00200000:2.0:1789760795.729430:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.729438:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.729441:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24219648, 24223744) 0 0 00000080:00200000:2.0:1789760795.729444:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.729456:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5913, 5913] 00020000:00200000:2.0:1789760795.729462:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24219648 chunk: [24219648, 24223744) 24223744, 4096 00020000:00200000:2.0:1789760795.729466:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.729481:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5913, 5913] 00000008:00010000:2.0:1789760795.729494:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.729499:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.729508:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24219648, 24223744) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.729518:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24219648=0x1719000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.729556:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.729558:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24223744 00000020:00010000:2.0:1789760795.729564:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.729567:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.729570:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.729573:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.729576:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.729591:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.729596:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.729609:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50accc0 00000008:00100000:2.0:1789760795.729673:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24219648<>24223744 00000080:00200000:2.0:1789760795.785499:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5913, 5913] 00000080:00200000:2.0:1789760795.785513:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.785518:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.785531:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24223744 i_blocks 49159 00000020:00200000:2.0:1789760795.785534:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.785543:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.785570:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24223744, count: 4096 00000080:00200000:2.0:1789760795.785576:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24223744, bytes: 4096 00000080:00200000:2.0:1789760795.785593:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.785600:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.785602:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24223744, 24227840) 0 0 00000080:00200000:2.0:1789760795.785606:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.785619:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5914, 5914] 00020000:00200000:2.0:1789760795.785686:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24223744 chunk: [24223744, 24227840) 24227840, 4096 00020000:00200000:2.0:1789760795.785691:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.785708:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5914, 5914] 00000008:00010000:2.0:1789760795.785719:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.785723:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.785731:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24223744, 24227840) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.785740:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24223744=0x171a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.785821:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.785823:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24227840 00000020:00010000:2.0:1789760795.785828:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.785831:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.785833:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.785835:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.785837:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.785842:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.785846:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.785857:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760795.785929:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 24223744<>24227840 00000080:00200000:2.0:1789760795.820245:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5914, 5914] 00000080:00200000:2.0:1789760795.820261:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.820266:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.820279:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24227840 i_blocks 49159 00000020:00200000:2.0:1789760795.820283:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.820294:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.820324:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24227840, count: 4096 00000080:00200000:2.0:1789760795.820331:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24227840, bytes: 4096 00000080:00200000:2.0:1789760795.820341:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.820351:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.820353:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24227840, 24231936) 0 0 00000080:00200000:2.0:1789760795.820356:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.820369:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5915, 5915] 00020000:00200000:2.0:1789760795.820400:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24227840 chunk: [24227840, 24231936) 24231936, 4096 00020000:00200000:2.0:1789760795.820405:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.820421:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5915, 5915] 00000008:00010000:2.0:1789760795.820435:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.820441:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.820450:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24227840, 24231936) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.820460:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24227840=0x171b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.820505:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.820508:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24231936 00000020:00010000:2.0:1789760795.820514:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.820518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.820521:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.820524:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.820527:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.820534:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.820540:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.820555:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760795.821097:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 24227840<>24231936 00000080:00200000:2.0:1789760795.882011:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5915, 5915] 00000080:00200000:2.0:1789760795.882032:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.882039:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.882054:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24231936 i_blocks 49159 00000020:00200000:2.0:1789760795.882366:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.882413:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.882441:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24231936, count: 4096 00000080:00200000:2.0:1789760795.882448:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24231936, bytes: 4096 00000080:00200000:2.0:1789760795.882459:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.882466:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.882468:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24231936, 24236032) 0 0 00000080:00200000:2.0:1789760795.882472:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.882484:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5916, 5916] 00020000:00200000:2.0:1789760795.882491:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24231936 chunk: [24231936, 24236032) 24236032, 4096 00020000:00200000:2.0:1789760795.882495:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.882512:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5916, 5916] 00000008:00010000:2.0:1789760795.882527:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.882532:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.882541:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24231936, 24236032) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.882552:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24231936=0x171c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.882810:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.882814:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24236032 00000020:00010000:2.0:1789760795.882821:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.882825:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.882828:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.882832:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.882835:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.882842:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.882848:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.882868:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760795.882952:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 24231936<>24236032 00000080:00200000:2.0:1789760795.950765:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5916, 5916] 00000080:00200000:2.0:1789760795.950783:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.950789:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.950803:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24236032 i_blocks 49159 00000020:00200000:2.0:1789760795.950807:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.950818:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.950849:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24236032, count: 4096 00000080:00200000:2.0:1789760795.950855:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24236032, bytes: 4096 00000080:00200000:2.0:1789760795.950865:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.950934:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.950936:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24236032, 24240128) 0 0 00000080:00200000:2.0:1789760795.950940:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.950958:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5917, 5917] 00020000:00200000:2.0:1789760795.950966:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24236032 chunk: [24236032, 24240128) 24240128, 4096 00020000:00200000:2.0:1789760795.950970:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.950988:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5917, 5917] 00000008:00010000:2.0:1789760795.951004:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.951010:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.951021:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24236032, 24240128) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.951034:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24236032=0x171d000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.951085:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.951088:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24240128 00000020:00010000:2.0:1789760795.951096:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.951099:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.951102:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.951105:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.951108:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.951114:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.951120:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.951136:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760795.951238:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020a80 - object 0x280000401:3 offset 24236032<>24240128 00000080:00200000:2.0:1789760795.985490:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5917, 5917] 00000080:00200000:2.0:1789760795.985508:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760795.985515:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760795.985531:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24240128 i_blocks 49159 00000020:00200000:2.0:1789760795.985535:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760795.985545:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760795.985599:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24240128, count: 4096 00000080:00200000:2.0:1789760795.985607:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24240128, bytes: 4096 00000080:00200000:2.0:1789760795.985616:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760795.985633:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760795.985635:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24240128, 24244224) 0 0 00000080:00200000:2.0:1789760795.985640:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760795.985655:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5918, 5918] 00020000:00200000:2.0:1789760795.985663:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24240128 chunk: [24240128, 24244224) 24244224, 4096 00020000:00200000:2.0:1789760795.985667:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760795.985685:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5918, 5918] 00000008:00010000:2.0:1789760795.985704:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760795.985710:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760795.985722:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24240128, 24244224) append 0 append lockpos 0 00000080:00200000:2.0:1789760795.985737:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24240128=0x171e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760795.985798:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760795.985801:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24244224 00000020:00010000:2.0:1789760795.985809:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760795.985813:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.985816:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760795.985820:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760795.985823:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760795.985831:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760795.985837:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760795.985861:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ace40 00000008:00100000:2.0:1789760795.985973:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8020380 - object 0x280000401:3 offset 24240128<>24244224 00000080:00200000:2.0:1789760796.019860:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5918, 5918] 00000080:00200000:2.0:1789760796.019877:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.019884:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.019900:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24244224 i_blocks 49159 00000020:00200000:2.0:1789760796.019905:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.019916:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.019952:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24244224, count: 4096 00000080:00200000:2.0:1789760796.019960:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24244224, bytes: 4096 00000080:00200000:2.0:1789760796.019970:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.019981:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.019984:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24244224, 24248320) 0 0 00000080:00200000:2.0:1789760796.019988:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.020005:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5919, 5919] 00020000:00200000:2.0:1789760796.020013:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24244224 chunk: [24244224, 24248320) 24248320, 4096 00020000:00200000:2.0:1789760796.020017:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.020036:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5919, 5919] 00000008:00010000:2.0:1789760796.020055:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.020061:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.020074:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24244224, 24248320) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.020091:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24244224=0x171f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.020152:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.020155:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24248320 00000020:00010000:2.0:1789760796.020165:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.020169:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.020183:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.020188:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.020192:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.020200:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.020207:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.020224:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50aca80 00000008:00100000:2.0:1789760796.020331:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023800 - object 0x280000401:3 offset 24244224<>24248320 00000080:00200000:2.0:1789760796.056247:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5919, 5919] 00000080:00200000:2.0:1789760796.056265:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.056272:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.056288:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24248320 i_blocks 49159 00000020:00200000:2.0:1789760796.056292:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.056302:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.056344:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24248320, count: 4096 00000080:00200000:2.0:1789760796.056352:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24248320, bytes: 4096 00000080:00200000:2.0:1789760796.056363:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.056374:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.056405:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24248320, 24252416) 0 0 00000080:00200000:2.0:1789760796.056409:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.056426:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5920, 5920] 00020000:00200000:2.0:1789760796.056434:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24248320 chunk: [24248320, 24252416) 24252416, 4096 00020000:00200000:2.0:1789760796.056439:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.056459:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5920, 5920] 00000008:00010000:2.0:1789760796.056482:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.056488:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.056500:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24248320, 24252416) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.056514:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24248320=0x1720000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.056575:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.056589:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24252416 00000020:00010000:2.0:1789760796.056598:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.056602:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.056605:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.056609:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.056612:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.056620:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.056627:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.056644:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760796.056748:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24248320<>24252416 00000100:00100000:2.0:1789760796.091777:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973240 gen 3 00000008:00010000:2.0:1789760796.091826:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac180 00000020:00200000:2.0:1789760796.091835:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.091847:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.091871:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac180 00000100:00100000:2.0:1789760796.091899:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507395712:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760796.092219:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5920, 5920] 00000080:00200000:1.0:1789760796.092236:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760796.092242:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760796.092257:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24252416 i_blocks 49159 00000020:00200000:1.0:1789760796.092261:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.092270:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760796.092304:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24252416, count: 4096 00000080:00200000:1.0:1789760796.092311:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24252416, bytes: 4096 00000080:00200000:1.0:1789760796.092320:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760796.092329:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760796.092331:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24252416, 24256512) 0 0 00000080:00200000:1.0:1789760796.092336:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760796.092351:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5921, 5921] 00020000:00200000:1.0:1789760796.092358:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24252416 chunk: [24252416, 24256512) 24256512, 4096 00020000:00200000:1.0:1789760796.092362:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760796.092407:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5921, 5921] 00000008:00010000:1.0:1789760796.092423:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760796.092429:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760796.092439:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24252416, 24256512) append 0 append lockpos 0 00000080:00200000:1.0:1789760796.092452:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24252416=0x1721000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760796.092502:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760796.092505:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24256512 00000020:00010000:1.0:1789760796.092512:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760796.092515:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.092518:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.092522:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760796.092525:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760796.092533:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760796.092540:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.092557:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760796.092663:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169500 - object 0x280000401:3 offset 24252416<>24256512 00000100:00100000:0.0:1789760796.092729:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507395840:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760796.123414:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760796.123431:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.123444:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.123475:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760796.123567:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507395840:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760796.123751:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5921, 5921] 00000080:00200000:2.0:1789760796.123768:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.123775:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.123790:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24256512 i_blocks 49159 00000020:00200000:2.0:1789760796.123793:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.123803:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.123840:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24256512, count: 4096 00000080:00200000:2.0:1789760796.123847:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24256512, bytes: 4096 00000080:00200000:2.0:1789760796.123860:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.124085:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.124088:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24256512, 24260608) 0 0 00000080:00200000:2.0:1789760796.124093:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.124110:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5922, 5922] 00020000:00200000:2.0:1789760796.124118:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24256512 chunk: [24256512, 24260608) 24260608, 4096 00020000:00200000:2.0:1789760796.124123:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.124715:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5922, 5922] 00000008:00010000:2.0:1789760796.126082:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.126095:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.126115:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24256512, 24260608) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.126136:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24256512=0x1722000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.126221:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.126225:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24260608 00000020:00010000:2.0:1789760796.126233:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.126337:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.126341:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.126344:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.126347:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.126359:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.126369:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.127688:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760796.127824:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24256512<>24260608 00000100:00100000:2.0:1789760796.127899:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760796.127934:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507395968:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.187327:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973242 gen 3 00000008:00010000:2.0:1789760796.187413:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac180 00000020:00200000:2.0:1789760796.187422:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.187433:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.188158:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac180 00000100:00100000:2.0:1789760796.188197:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507395968:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760796.193110:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5922, 5922] 00000080:00200000:1.0:1789760796.193213:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760796.193221:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760796.193241:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24260608 i_blocks 49159 00000020:00200000:1.0:1789760796.193247:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.193328:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760796.193365:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24260608, count: 4096 00000080:00200000:1.0:1789760796.193375:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24260608, bytes: 4096 00000080:00200000:1.0:1789760796.193969:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760796.193981:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760796.193984:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24260608, 24264704) 0 0 00000080:00200000:1.0:1789760796.193989:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760796.194008:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5923, 5923] 00020000:00200000:1.0:1789760796.194017:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24260608 chunk: [24260608, 24264704) 24264704, 4096 00020000:00200000:1.0:1789760796.194086:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760796.194115:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5923, 5923] 00000008:00010000:1.0:1789760796.194207:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760796.194214:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760796.194229:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24260608, 24264704) append 0 append lockpos 0 00000080:00200000:1.0:1789760796.194245:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24260608=0x1723000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760796.194752:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760796.194755:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24264704 00000020:00010000:1.0:1789760796.194766:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760796.194769:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.194773:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.194776:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760796.194779:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760796.194792:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760796.194799:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.194833:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760796.196124:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 24260608<>24264704 00000100:00100000:0.0:1789760796.196331:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760796.199663:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507396096:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760796.251762:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973243 gen 3 00000008:00010000:1.0:1789760796.251811:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760796.251818:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.251829:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.251850:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760796.251878:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507396096:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760796.252139:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5923, 5923] 00000080:00200000:0.0:1789760796.252159:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760796.252166:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760796.252181:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24264704 i_blocks 49159 00000020:00200000:0.0:1789760796.252185:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760796.252195:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760796.252227:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24264704, count: 4096 00000080:00200000:0.0:1789760796.252235:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24264704, bytes: 4096 00000080:00200000:0.0:1789760796.252245:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760796.252254:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760796.252257:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24264704, 24268800) 0 0 00000080:00200000:0.0:1789760796.252262:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760796.252277:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5924, 5924] 00020000:00200000:0.0:1789760796.252285:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24264704 chunk: [24264704, 24268800) 24268800, 4096 00020000:00200000:0.0:1789760796.252289:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760796.252308:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5924, 5924] 00000008:00010000:0.0:1789760796.252326:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760796.252332:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760796.252343:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24264704, 24268800) append 0 append lockpos 0 00000080:00200000:0.0:1789760796.252356:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24264704=0x1724000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760796.252442:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760796.252445:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24268800 00000020:00010000:0.0:1789760796.252454:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760796.252457:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760796.252460:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760796.252464:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760796.252468:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760796.252476:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760796.252482:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.252570:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000008:00100000:1.0:1789760796.253007:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 24264704<>24268800 00000100:00100000:1.0:1789760796.253274:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507396224:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760796.308796:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973244 gen 3 00000008:00010000:1.0:1789760796.308847:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000020:00200000:1.0:1789760796.308856:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.308867:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.308888:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7cc0 00000080:00200000:0.0:1789760796.308973:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5924, 5924] 00000080:00200000:0.0:1789760796.308992:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760796.308998:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760796.309014:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24268800 i_blocks 49159 00000020:00200000:0.0:1789760796.309027:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760796.309038:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760796.309074:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24268800, count: 4096 00000080:00200000:0.0:1789760796.309193:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24268800, bytes: 4096 00000080:00200000:0.0:1789760796.309206:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760796.309214:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760796.309216:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24268800, 24272896) 0 0 00000080:00200000:0.0:1789760796.309221:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760796.309238:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5925, 5925] 00020000:00200000:0.0:1789760796.309247:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24268800 chunk: [24268800, 24272896) 24272896, 4096 00020000:00200000:0.0:1789760796.309252:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760796.309275:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5925, 5925] 00000008:00010000:0.0:1789760796.309295:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760796.309301:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760796.309313:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24268800, 24272896) append 0 append lockpos 0 00000080:00200000:0.0:1789760796.309328:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24268800=0x1725000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760796.309416:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760796.309420:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24272896 00000020:00010000:0.0:1789760796.309429:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760796.309433:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760796.309436:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760796.309440:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760796.309443:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760796.309452:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760796.309459:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760796.309480:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760796.309790:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507396224:192.168.203.141@tcp:4:dd.0 00000008:00100000:0.0:1789760796.310340:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b9c00 - object 0x280000401:3 offset 24268800<>24272896 00000100:00100000:0.0:1789760796.310438:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507396352:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760796.353342:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973245 gen 3 00000008:00010000:1.0:1789760796.353430:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000020:00200000:1.0:1789760796.353441:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.353458:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.353487:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7c00 00000100:00100000:1.0:1789760796.353517:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507396352:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760796.354223:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5925, 5925] 00000080:00200000:2.0:1789760796.354244:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.354250:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.354267:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24272896 i_blocks 49159 00000020:00200000:2.0:1789760796.354271:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.354282:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.354318:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24272896, count: 4096 00000080:00200000:2.0:1789760796.354326:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24272896, bytes: 4096 00000080:00200000:2.0:1789760796.354335:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.354344:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.354346:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24272896, 24276992) 0 0 00000080:00200000:2.0:1789760796.354351:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.354366:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5926, 5926] 00020000:00200000:2.0:1789760796.354374:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24272896 chunk: [24272896, 24276992) 24276992, 4096 00020000:00200000:2.0:1789760796.354406:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.354427:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5926, 5926] 00000008:00010000:2.0:1789760796.354446:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.354452:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.354465:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24272896, 24276992) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.354479:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24272896=0x1726000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.354540:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.354543:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24276992 00000020:00010000:2.0:1789760796.354551:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.354554:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.354557:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.354560:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.354563:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.354570:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.354590:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.354609:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760796.354885:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24272896<>24276992 00000008:00010000:2.0:1789760796.404759:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac180 00000020:00200000:2.0:1789760796.404768:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.404780:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.404803:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac180 00000100:00100000:2.0:1789760796.404834:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507396480:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760796.404988:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5926, 5926] 00000080:00200000:1.0:1789760796.405004:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760796.405011:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760796.405027:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24276992 i_blocks 49159 00000020:00200000:1.0:1789760796.405031:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.405042:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760796.405076:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24276992, count: 4096 00000080:00200000:1.0:1789760796.405083:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24276992, bytes: 4096 00000080:00200000:1.0:1789760796.405094:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760796.405103:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760796.405106:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24276992, 24281088) 0 0 00000080:00200000:1.0:1789760796.405111:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760796.405129:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5927, 5927] 00020000:00200000:1.0:1789760796.405137:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24276992 chunk: [24276992, 24281088) 24281088, 4096 00020000:00200000:1.0:1789760796.405142:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760796.405162:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5927, 5927] 00000008:00010000:1.0:1789760796.405181:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760796.405187:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760796.405199:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24276992, 24281088) append 0 append lockpos 0 00000080:00200000:1.0:1789760796.405213:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24276992=0x1727000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760796.405273:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760796.405276:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24281088 00000020:00010000:1.0:1789760796.405284:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760796.405288:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.405291:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.405295:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760796.405298:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760796.405307:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760796.405314:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.405330:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760796.405570:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8168000 - object 0x280000401:3 offset 24276992<>24281088 00000100:00100000:0.0:1789760796.405656:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507396608:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760796.445006:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760796.445083:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.445095:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.445170:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760796.445203:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507396608:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760796.445636:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5927, 5927] 00000080:00200000:2.0:1789760796.445653:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.445659:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.445674:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24281088 i_blocks 49159 00000020:00200000:2.0:1789760796.445678:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.445877:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.445922:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24281088, count: 4096 00000080:00200000:2.0:1789760796.445930:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24281088, bytes: 4096 00000080:00200000:2.0:1789760796.445941:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.445951:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.445953:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24281088, 24285184) 0 0 00000080:00200000:2.0:1789760796.445958:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.445975:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5928, 5928] 00020000:00200000:2.0:1789760796.445984:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24281088 chunk: [24281088, 24285184) 24285184, 4096 00020000:00200000:2.0:1789760796.445988:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.446008:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5928, 5928] 00000008:00010000:2.0:1789760796.446025:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.446032:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.446043:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24281088, 24285184) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.446057:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24281088=0x1728000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.446121:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.446124:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24285184 00000020:00010000:2.0:1789760796.446133:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.446137:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.446140:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.446144:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.446146:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.446155:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.446161:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.446182:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760796.446290:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24281088<>24285184 00000100:00100000:2.0:1789760796.446339:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760796.446359:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507396736:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.488710:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973248 gen 3 00000008:00010000:2.0:1789760796.488775:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac180 00000020:00200000:2.0:1789760796.488783:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.488792:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.488812:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac180 00000100:00100000:2.0:1789760796.488840:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507396736:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760796.524633:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5929, 5929] 00000080:00200000:2.0:1789760796.524651:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.524657:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.524674:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24289280 i_blocks 49159 00000020:00200000:2.0:1789760796.524678:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.524688:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.524721:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24289280, count: 4096 00000080:00200000:2.0:1789760796.524729:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24289280, bytes: 4096 00000080:00200000:2.0:1789760796.524739:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.524748:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.524751:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24289280, 24293376) 0 0 00000080:00200000:2.0:1789760796.524755:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.524771:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5930, 5930] 00020000:00200000:2.0:1789760796.524780:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24289280 chunk: [24289280, 24293376) 24293376, 4096 00020000:00200000:2.0:1789760796.524784:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.524804:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5930, 5930] 00000008:00010000:2.0:1789760796.524825:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.524831:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.524844:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24289280, 24293376) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.524859:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24289280=0x172a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.524929:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.524933:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24293376 00000020:00010000:2.0:1789760796.524943:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.524947:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.524951:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.524955:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.524958:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.524966:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.524973:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.524988:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760796.525267:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24289280<>24293376 00000100:00100000:2.0:1789760796.525412:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760796.525433:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507396992:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.586125:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973250 gen 3 00000008:00010000:2.0:1789760796.586177:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac180 00000020:00200000:2.0:1789760796.586185:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.586197:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.586275:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac180 00000100:00100000:2.0:1789760796.586308:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507396992:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760796.586437:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5930, 5930] 00000080:00200000:1.0:1789760796.586454:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760796.586461:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760796.586477:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24293376 i_blocks 49159 00000020:00200000:1.0:1789760796.586481:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.586492:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760796.586528:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24293376, count: 4096 00000080:00200000:1.0:1789760796.586535:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24293376, bytes: 4096 00000080:00200000:1.0:1789760796.586546:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760796.586555:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760796.586558:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24293376, 24297472) 0 0 00000080:00200000:1.0:1789760796.586562:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760796.586588:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5931, 5931] 00020000:00200000:1.0:1789760796.586596:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24293376 chunk: [24293376, 24297472) 24297472, 4096 00020000:00200000:1.0:1789760796.586601:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760796.586687:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5931, 5931] 00000008:00010000:1.0:1789760796.586707:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760796.586714:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760796.586725:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24293376, 24297472) append 0 append lockpos 0 00000080:00200000:1.0:1789760796.586740:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24293376=0x172b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760796.586801:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760796.586804:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24297472 00000020:00010000:1.0:1789760796.586813:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760796.586817:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.586820:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.586823:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760796.586826:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760796.586836:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760796.586842:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.586860:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760796.587015:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24293376<>24297472 00000100:00100000:0.0:1789760796.587139:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507397120:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760796.633430:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973251 gen 3 00000008:00010000:1.0:1789760796.633484:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760796.633493:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.633505:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.633532:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760796.633562:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507397120:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760796.634013:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5931, 5931] 00000080:00200000:2.0:1789760796.634032:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.634038:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.634054:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24297472 i_blocks 49159 00000020:00200000:2.0:1789760796.634057:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.634069:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.634102:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24297472, count: 4096 00000080:00200000:2.0:1789760796.634109:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24297472, bytes: 4096 00000080:00200000:2.0:1789760796.634120:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.634129:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.634132:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24297472, 24301568) 0 0 00000080:00200000:2.0:1789760796.634138:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.634155:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5932, 5932] 00020000:00200000:2.0:1789760796.634164:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24297472 chunk: [24297472, 24301568) 24301568, 4096 00020000:00200000:2.0:1789760796.634169:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.634188:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5932, 5932] 00000008:00010000:2.0:1789760796.634208:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.634213:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.634224:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24297472, 24301568) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.634238:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24297472=0x172c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.634302:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.634305:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24301568 00000020:00010000:2.0:1789760796.634313:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.634317:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.634321:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.634325:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.634328:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.634336:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.634343:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.634359:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760796.637855:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24297472<>24301568 00000100:00100000:2.0:1789760796.637963:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507397248:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760796.705617:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac180 00000020:00200000:2.0:1789760796.705635:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.705647:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.705669:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac180 00000100:00100000:2.0:1789760796.705801:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507397248:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.706533:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507397376:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.738091:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973253 gen 3 00000008:00010000:2.0:1789760796.738166:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c240 00000020:00200000:2.0:1789760796.738177:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.738188:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.738217:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c240 00000100:00100000:2.0:1789760796.738257:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507397376:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.739188:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760796.739208:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507397504:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.771250:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973254 gen 3 00000008:00010000:2.0:1789760796.771306:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000020:00200000:2.0:1789760796.771315:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.771327:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.771441:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5c9c0 00000100:00100000:2.0:1789760796.771472:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507397504:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760796.833529:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5935, 5935] 00000080:00200000:1.0:1789760796.833545:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760796.833551:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760796.833569:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24313856 i_blocks 49159 00000020:00200000:1.0:1789760796.833585:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.833595:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760796.833629:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24313856, count: 4096 00000080:00200000:1.0:1789760796.833636:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24313856, bytes: 4096 00000080:00200000:1.0:1789760796.833647:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760796.833656:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760796.833659:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24313856, 24317952) 0 0 00000080:00200000:1.0:1789760796.833663:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760796.833680:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5936, 5936] 00020000:00200000:1.0:1789760796.833739:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24313856 chunk: [24313856, 24317952) 24317952, 4096 00020000:00200000:1.0:1789760796.833744:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760796.833766:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5936, 5936] 00000008:00010000:1.0:1789760796.833784:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760796.833789:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760796.833802:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24313856, 24317952) append 0 append lockpos 0 00000080:00200000:1.0:1789760796.833818:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24313856=0x1730000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760796.833878:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760796.833882:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24317952 00000020:00010000:1.0:1789760796.833891:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760796.833894:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.833898:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.833902:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760796.833905:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760796.833914:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760796.833921:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.833937:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760796.834049:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24313856<>24317952 00000100:00100000:0.0:1789760796.834124:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507397760:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760796.864890:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973256 gen 3 00000008:00010000:1.0:1789760796.864944:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760796.864953:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.864964:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.865057:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760796.865090:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507397760:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.865989:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507397888:192.168.203.141@tcp:4:dd.0 00000008:00010000:2.0:1789760796.899498:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c5e5ce40 00000020:00200000:2.0:1789760796.899515:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.899528:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760796.899550:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c5e5ce40 00000080:00200000:1.0:1789760796.899621:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5937, 5937] 00000080:00200000:1.0:1789760796.899640:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760796.899647:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760796.899664:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24322048 i_blocks 49159 00000020:00200000:1.0:1789760796.899668:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.899679:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000100:00100000:2.0:1789760796.899705:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507397888:192.168.203.141@tcp:4:dd.0 00000080:00200008:1.0:1789760796.899723:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24322048, count: 4096 00000080:00200000:1.0:1789760796.899732:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24322048, bytes: 4096 00000080:00200000:1.0:1789760796.899742:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760796.899751:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760796.899753:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24322048, 24326144) 0 0 00000080:00200000:1.0:1789760796.899758:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760796.899777:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5938, 5938] 00020000:00200000:1.0:1789760796.899787:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24322048 chunk: [24322048, 24326144) 24326144, 4096 00020000:00200000:1.0:1789760796.899793:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760796.899813:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5938, 5938] 00000008:00010000:1.0:1789760796.899831:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760796.899836:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760796.899846:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24322048, 24326144) append 0 append lockpos 0 00000080:00200000:1.0:1789760796.899857:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24322048=0x1732000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760796.899909:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760796.899912:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24326144 00000020:00010000:1.0:1789760796.899921:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760796.899924:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.899927:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.899931:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760796.899933:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760796.899942:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760796.899949:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.899970:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760796.900207:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24322048<>24326144 00000100:00100000:0.0:1789760796.900282:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398016:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760796.963763:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973258 gen 3 00000008:00010000:1.0:1789760796.963829:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760796.963836:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760796.964010:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760796.964035:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760796.964171:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398016:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760796.964511:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5938, 5938] 00000080:00200000:2.0:1789760796.964529:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760796.964535:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760796.964551:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24326144 i_blocks 49159 00000020:00200000:2.0:1789760796.964555:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760796.964566:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760796.964613:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24326144, count: 4096 00000080:00200000:2.0:1789760796.964621:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24326144, bytes: 4096 00000080:00200000:2.0:1789760796.964636:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760796.964645:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760796.964647:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24326144, 24330240) 0 0 00000080:00200000:2.0:1789760796.964653:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760796.964669:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5939, 5939] 00020000:00200000:2.0:1789760796.964676:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24326144 chunk: [24326144, 24330240) 24330240, 4096 00020000:00200000:2.0:1789760796.964681:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760796.964699:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5939, 5939] 00000008:00010000:2.0:1789760796.964716:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760796.964730:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760796.964742:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24326144, 24330240) append 0 append lockpos 0 00000080:00200000:2.0:1789760796.964758:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24326144=0x1733000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760796.964816:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760796.964819:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24330240 00000020:00010000:2.0:1789760796.964827:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760796.964830:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.964833:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760796.964836:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760796.964839:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760796.964857:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760796.965077:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760796.965099:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760796.965210:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24326144<>24330240 00000100:00100000:2.0:1789760796.965263:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:2.0:1789760796.965294:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507398144:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760796.998511:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973259 gen 3 00000008:00010000:2.0:1789760796.998560:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac180 00000020:00200000:2.0:1789760796.998578:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760796.998589:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.998657:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5939, 5939] 00000080:00200000:1.0:1789760796.998675:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760796.998681:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760796.998696:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24330240 i_blocks 49159 00000020:00200000:1.0:1789760796.998700:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760796.998710:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760796.998744:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24330240, count: 4096 00000080:00200000:1.0:1789760796.998753:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24330240, bytes: 4096 00000080:00200000:1.0:1789760796.998763:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760796.998773:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760796.998775:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24330240, 24334336) 0 0 00000080:00200000:1.0:1789760796.998780:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760796.998797:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5940, 5940] 00020000:00200000:1.0:1789760796.998804:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24330240 chunk: [24330240, 24334336) 24334336, 4096 00020000:00200000:1.0:1789760796.998808:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760796.998829:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5940, 5940] 00000008:00010000:1.0:1789760796.998847:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760796.998853:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760796.998930:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24330240, 24334336) append 0 append lockpos 0 00000008:00010000:2.0:1789760796.999279:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac180 00000100:00100000:2.0:1789760796.999318:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507398144:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760796.999443:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24330240=0x1734000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760796.999658:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760796.999661:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24334336 00000020:00010000:1.0:1789760796.999672:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760796.999676:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.999679:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760796.999683:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760796.999686:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760796.999695:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760796.999704:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760796.999736:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760796.999981:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24330240<>24334336 00000100:00100000:1.0:1789760797.000033:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760797.000054:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398272:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760797.055075:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760797.055093:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.055106:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.055130:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760797.055162:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398272:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760797.056942:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5940, 5940] 00000080:00200000:0.0:1789760797.056966:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760797.056973:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760797.056990:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24334336 i_blocks 49159 00000020:00200000:0.0:1789760797.056996:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760797.057011:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760797.057049:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24334336, count: 4096 00000080:00200000:0.0:1789760797.057057:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24334336, bytes: 4096 00000080:00200000:0.0:1789760797.057072:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760797.057083:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760797.057085:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24334336, 24338432) 0 0 00000080:00200000:0.0:1789760797.057090:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760797.057107:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5941, 5941] 00020000:00200000:0.0:1789760797.057115:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24334336 chunk: [24334336, 24338432) 24338432, 4096 00020000:00200000:0.0:1789760797.057120:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760797.057143:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5941, 5941] 00000008:00010000:0.0:1789760797.057162:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760797.057168:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760797.057179:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24334336, 24338432) append 0 append lockpos 0 00000080:00200000:0.0:1789760797.057195:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24334336=0x1735000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760797.058054:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760797.058062:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24338432 00000020:00010000:0.0:1789760797.058074:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760797.058078:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.058081:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.058084:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760797.058087:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760797.058099:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760797.058109:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760797.058140:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7540 00000008:00100000:0.0:1789760797.058254:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25bbb80 - object 0x280000401:3 offset 24334336<>24338432 00000100:00100000:0.0:1789760797.058304:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00100000:0.0:1789760797.058331:0:2416:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507398400:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760797.126318:0:2416:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973260 gen 3 00000100:00100000:1.0:1789760797.126362:0:2416:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25bbb80 x1876699507398400/t4294973261(4294973261) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760813 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760797.126465:0:2416:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7540 00000020:00200000:1.0:1789760797.126473:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.126487:0:2416:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.126502:0:2416:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7540 00000100:00100000:1.0:1789760797.126525:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:7fdb621d-3376-4597-a0ab-d89800273221:2416:1876699507398400:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760797.126780:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5941, 5941] 00000080:00200000:0.0:1789760797.126798:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760797.126804:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760797.126820:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24338432 i_blocks 49159 00000020:00200000:0.0:1789760797.126824:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760797.126833:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760797.126867:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24338432, count: 4096 00000080:00200000:0.0:1789760797.126874:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24338432, bytes: 4096 00000080:00200000:0.0:1789760797.126885:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760797.126905:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760797.126908:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24338432, 24342528) 0 0 00000080:00200000:0.0:1789760797.126914:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760797.126931:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5942, 5942] 00020000:00200000:0.0:1789760797.126941:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24338432 chunk: [24338432, 24342528) 24342528, 4096 00020000:00200000:0.0:1789760797.126946:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760797.126966:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5942, 5942] 00000008:00010000:0.0:1789760797.126984:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760797.126990:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760797.127002:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24338432, 24342528) append 0 append lockpos 0 00000080:00200000:0.0:1789760797.127084:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24338432=0x1736000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760797.127147:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760797.127149:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24342528 00000020:00010000:0.0:1789760797.127159:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760797.127162:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.127165:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.127169:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760797.127172:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760797.127180:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760797.127188:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760797.127207:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7600 00000008:00100000:0.0:1789760797.127318:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8000 - object 0x280000401:3 offset 24338432<>24342528 00000100:00100000:1.0:1789760797.127508:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760797.127532:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398528:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760797.160162:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973262 gen 3 00000008:00010000:1.0:1789760797.160233:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7600 00000020:00200000:1.0:1789760797.160242:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.160255:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.160320:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7600 00000100:00100000:1.0:1789760797.160354:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398528:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760797.160539:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5942, 5942] 00000080:00200000:0.0:1789760797.160556:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760797.160563:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760797.160589:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24342528 i_blocks 49159 00000020:00200000:0.0:1789760797.160593:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760797.160603:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760797.160706:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24342528, count: 4096 00000080:00200000:0.0:1789760797.160714:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24342528, bytes: 4096 00000080:00200000:0.0:1789760797.160726:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760797.160735:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760797.160738:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24342528, 24346624) 0 0 00000080:00200000:0.0:1789760797.160742:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760797.160761:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5943, 5943] 00020000:00200000:0.0:1789760797.160770:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24342528 chunk: [24342528, 24346624) 24346624, 4096 00020000:00200000:0.0:1789760797.160775:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760797.160795:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5943, 5943] 00000008:00010000:0.0:1789760797.160813:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760797.160819:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760797.160830:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24342528, 24346624) append 0 append lockpos 0 00000080:00200000:0.0:1789760797.160843:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24342528=0x1737000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760797.160900:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760797.160903:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24346624 00000020:00010000:0.0:1789760797.160911:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760797.160915:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.160917:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.160921:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760797.160923:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760797.160931:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760797.160938:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760797.160955:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef7840 00000008:00100000:0.0:1789760797.161130:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25ba300 - object 0x280000401:3 offset 24342528<>24346624 00000100:00100000:1.0:1789760797.161199:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398656:192.168.203.141@tcp:4:dd.0 00000008:00010000:1.0:1789760797.207499:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef7840 00000020:00200000:1.0:1789760797.207515:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.207528:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.207604:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef7840 00000100:00100000:1.0:1789760797.207642:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398656:192.168.203.141@tcp:4:dd.0 00000080:00200000:0.0:1789760797.211764:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5943, 5943] 00000080:00200000:0.0:1789760797.211793:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760797.211800:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760797.211820:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24346624 i_blocks 49159 00000020:00200000:0.0:1789760797.211827:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760797.211845:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760797.211880:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24346624, count: 4096 00000080:00200000:0.0:1789760797.211889:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24346624, bytes: 4096 00000080:00200000:0.0:1789760797.211904:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760797.211914:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760797.211916:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24346624, 24350720) 0 0 00000080:00200000:0.0:1789760797.211920:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:0.0:1789760797.211939:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5944, 5944] 00020000:00200000:0.0:1789760797.211947:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24346624 chunk: [24346624, 24350720) 24350720, 4096 00020000:00200000:0.0:1789760797.211952:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:0.0:1789760797.211974:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5944, 5944] 00000008:00010000:0.0:1789760797.211995:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:0.0:1789760797.212001:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c51e1e30, osc lock ffff8d25d1b651d0, flags 0x0 00000080:00200000:0.0:1789760797.212725:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24346624, 24350720) append 0 append lockpos 0 00000080:00200000:0.0:1789760797.212747:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24346624=0x1738000, pages 1 (max 131072), lockless, parallel 00000020:00200000:0.0:1789760797.212823:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:0.0:1789760797.212826:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24350720 00000020:00010000:0.0:1789760797.212836:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:0.0:1789760797.212839:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.212844:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c51e1e30 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:0.0:1789760797.212847:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:0.0:1789760797.212850:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c51e1e30 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:0.0:1789760797.212861:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:0.0:1789760797.212869:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:0.0:1789760797.212899:0:452:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000008:00100000:0.0:1789760797.213208:0:452:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25d25b8000 - object 0x280000401:3 offset 24346624<>24350720 00000100:00100000:1.0:1789760797.213273:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760797.214184:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398784:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760797.264286:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973263 gen 3 00000100:00100000:1.0:1789760797.264331:0:2417:0:(client.c:3169:ptlrpc_free_committed()) @@@ stopping search req@ffff8d25d25b8000 x1876699507398784/t4294973264(4294973264) o4->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:6/4 lens 4584/448 e 0 to 0 dl 1789760813 ref 3 fl Rpc:RQU/604/0 rc 0/0 job:'dd.0' uid:0 gid:0 projid:0 00000008:00010000:1.0:1789760797.264402:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000020:00200000:1.0:1789760797.264410:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.264421:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.264449:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c7ef73c0 00000100:00100000:1.0:1789760797.264473:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507398784:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760797.264602:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5944, 5944] 00000080:00200000:2.0:1789760797.264619:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.264625:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.264640:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24350720 i_blocks 49159 00000020:00200000:2.0:1789760797.264643:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.264653:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.264688:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24350720, count: 4096 00000080:00200000:2.0:1789760797.264695:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24350720, bytes: 4096 00000080:00200000:2.0:1789760797.264705:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.264714:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.264716:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24350720, 24354816) 0 0 00000080:00200000:2.0:1789760797.264721:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.264737:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5945, 5945] 00020000:00200000:2.0:1789760797.264745:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24350720 chunk: [24350720, 24354816) 24354816, 4096 00020000:00200000:2.0:1789760797.264750:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.264771:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5945, 5945] 00000008:00010000:2.0:1789760797.264789:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.264794:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.264805:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24350720, 24354816) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.264818:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24350720=0x1739000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.264873:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.264876:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24354816 00000020:00010000:2.0:1789760797.264885:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.264888:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.264892:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038a28/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.264895:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.264898:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.264906:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.264921:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.264938:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760797.265312:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022680 - object 0x280000401:3 offset 24350720<>24354816 00000080:00200000:0.0:1789760797.316250:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5945, 5945] 00000080:00200000:0.0:1789760797.316267:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:0.0:1789760797.316273:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:0.0:1789760797.316290:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24354816 i_blocks 49159 00000020:00200000:0.0:1789760797.316294:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:0.0:1789760797.316305:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:0.0:1789760797.316339:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24354816, count: 4096 00000080:00200000:0.0:1789760797.316346:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24354816, bytes: 4096 00000080:00200000:0.0:1789760797.316561:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:0.0:1789760797.316592:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:0.0:1789760797.316594:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24354816, 24358912) 0 0 00000080:00200000:0.0:1789760797.316599:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.316757:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5946, 5946] 00020000:00200000:2.0:1789760797.316766:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24354816 chunk: [24354816, 24358912) 24358912, 4096 00020000:00200000:2.0:1789760797.316771:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.316793:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5946, 5946] 00000008:00010000:2.0:1789760797.316812:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.316818:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.316830:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24354816, 24358912) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.316845:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24354816=0x173a000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.317269:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.317272:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24358912 00000020:00010000:2.0:1789760797.317282:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.317285:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.317288:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.317293:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.317295:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.317304:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.317312:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.317407:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760797.317519:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8022300 - object 0x280000401:3 offset 24354816<>24358912 00000080:00200000:2.0:1789760797.357257:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5946, 5946] 00000080:00200000:2.0:1789760797.357273:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.357279:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.357294:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24358912 i_blocks 49159 00000020:00200000:2.0:1789760797.357297:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.357306:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.357341:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24358912, count: 4096 00000080:00200000:2.0:1789760797.357348:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24358912, bytes: 4096 00000080:00200000:2.0:1789760797.357360:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.357370:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.357373:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24358912, 24363008) 0 0 00000080:00200000:2.0:1789760797.357405:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.357422:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5947, 5947] 00020000:00200000:2.0:1789760797.357430:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24358912 chunk: [24358912, 24363008) 24363008, 4096 00020000:00200000:2.0:1789760797.357434:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.357452:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5947, 5947] 00000008:00010000:2.0:1789760797.357469:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.357475:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.357486:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24358912, 24363008) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.357499:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24358912=0x173b000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.357675:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.357678:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24363008 00000020:00010000:2.0:1789760797.357687:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.357690:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.357693:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.357696:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.357699:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.357706:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.357713:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.357732:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760797.357836:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021880 - object 0x280000401:3 offset 24358912<>24363008 00000080:00200000:1.0:1789760797.400214:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5947, 5947] 00000080:00200000:1.0:1789760797.400237:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760797.400244:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760797.400261:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24363008 i_blocks 49159 00000020:00200000:1.0:1789760797.400266:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760797.400279:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760797.400313:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24363008, count: 4096 00000080:00200000:1.0:1789760797.400320:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24363008, bytes: 4096 00000080:00200000:1.0:1789760797.400334:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760797.400343:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760797.400345:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24363008, 24367104) 0 0 00000080:00200000:1.0:1789760797.400350:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760797.400366:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5948, 5948] 00020000:00200000:1.0:1789760797.400374:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24363008 chunk: [24363008, 24367104) 24367104, 4096 00020000:00200000:1.0:1789760797.400413:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760797.400435:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5948, 5948] 00000008:00010000:1.0:1789760797.400457:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760797.400463:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760797.400475:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24363008, 24367104) append 0 append lockpos 0 00000080:00200000:1.0:1789760797.400490:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24363008=0x173c000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760797.400555:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760797.400559:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24367104 00000020:00010000:1.0:1789760797.400578:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760797.400582:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760797.400585:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760797.400589:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760797.400592:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760797.400601:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760797.400608:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760797.400628:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760797.400738:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24363008<>24367104 00000100:00100000:0.0:1789760797.400889:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507399296:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760797.435698:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973268 gen 3 00000008:00010000:1.0:1789760797.435752:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760797.435761:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.435771:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.435837:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760797.435867:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507399296:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760797.470370:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5949, 5949] 00000080:00200000:2.0:1789760797.470421:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.470427:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.470443:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24371200 i_blocks 49159 00000020:00200000:2.0:1789760797.470447:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.470456:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.470490:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24371200, count: 4096 00000080:00200000:2.0:1789760797.470496:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24371200, bytes: 4096 00000080:00200000:2.0:1789760797.470507:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.470516:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.470519:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24371200, 24375296) 0 0 00000080:00200000:2.0:1789760797.470523:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.470540:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5950, 5950] 00020000:00200000:2.0:1789760797.470549:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24371200 chunk: [24371200, 24375296) 24375296, 4096 00020000:00200000:2.0:1789760797.470553:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.470586:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5950, 5950] 00000008:00010000:2.0:1789760797.470673:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.470681:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.470694:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24371200, 24375296) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.470710:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24371200=0x173e000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.470770:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.470773:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24375296 00000020:00010000:2.0:1789760797.470782:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.470785:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.470788:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.470792:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.470794:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.470801:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.470808:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.470825:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760797.470998:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 24371200<>24375296 00000100:00100000:2.0:1789760797.471151:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760797.471170:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507399552:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760797.506649:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973270 gen 3 00000008:00010000:2.0:1789760797.506717:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac600 00000020:00200000:2.0:1789760797.506726:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760797.506735:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760797.506760:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac600 00000100:00100000:2.0:1789760797.506790:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507399552:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760797.507016:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5950, 5950] 00000080:00200000:1.0:1789760797.507032:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760797.507038:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760797.507053:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24375296 i_blocks 49159 00000020:00200000:1.0:1789760797.507056:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760797.507066:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760797.507101:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24375296, count: 4096 00000080:00200000:1.0:1789760797.507108:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24375296, bytes: 4096 00000080:00200000:1.0:1789760797.507118:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760797.507126:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760797.507128:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24375296, 24379392) 0 0 00000080:00200000:1.0:1789760797.507132:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760797.507148:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5951, 5951] 00020000:00200000:1.0:1789760797.507157:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24375296 chunk: [24375296, 24379392) 24379392, 4096 00020000:00200000:1.0:1789760797.507162:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760797.507180:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5951, 5951] 00000008:00010000:1.0:1789760797.507197:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760797.507203:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760797.507216:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24375296, 24379392) append 0 append lockpos 0 00000080:00200000:1.0:1789760797.507231:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24375296=0x173f000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760797.507290:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760797.507294:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24379392 00000020:00010000:1.0:1789760797.507302:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760797.507305:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760797.507308:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c8038bd8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760797.507312:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760797.507316:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760797.507323:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760797.507330:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760797.507347:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760797.507496:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24375296<>24379392 00000100:00100000:1.0:1789760797.507531:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760797.507549:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507399680:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760797.533684:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973271 gen 3 00000008:00010000:1.0:1789760797.533739:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760797.533748:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.533760:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.533788:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760797.533821:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507399680:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760797.534088:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5951, 5951] 00000080:00200000:2.0:1789760797.534105:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.534111:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.534127:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24379392 i_blocks 49159 00000020:00200000:2.0:1789760797.534131:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.534141:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.534177:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24379392, count: 4096 00000080:00200000:2.0:1789760797.534184:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24379392, bytes: 4096 00000080:00200000:2.0:1789760797.534194:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.534204:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.534206:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24379392, 24383488) 0 0 00000080:00200000:2.0:1789760797.534211:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.534227:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5952, 5952] 00020000:00200000:2.0:1789760797.534236:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24379392 chunk: [24379392, 24383488) 24383488, 4096 00020000:00200000:2.0:1789760797.534240:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.534261:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5952, 5952] 00000008:00010000:2.0:1789760797.534281:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.534286:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.534298:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24379392, 24383488) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.534314:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24379392=0x1740000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.534486:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.534490:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24383488 00000020:00010000:2.0:1789760797.534498:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.534502:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.534505:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.534508:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.534511:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.534520:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.534527:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.534545:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac600 00000008:00100000:2.0:1789760797.534909:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 24379392<>24383488 00000080:00200000:2.0:1789760797.567929:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5952, 5952] 00000080:00200000:2.0:1789760797.567945:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.567951:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.567967:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24383488 i_blocks 49159 00000020:00200000:2.0:1789760797.567971:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.567981:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.568080:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24383488, count: 4096 00000080:00200000:2.0:1789760797.568088:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24383488, bytes: 4096 00000080:00200000:2.0:1789760797.568100:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.568110:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.568112:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24383488, 24387584) 0 0 00000080:00200000:2.0:1789760797.568117:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.568134:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5953, 5953] 00020000:00200000:2.0:1789760797.568194:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24383488 chunk: [24383488, 24387584) 24387584, 4096 00020000:00200000:2.0:1789760797.568199:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.568220:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5953, 5953] 00000008:00010000:2.0:1789760797.568239:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.568245:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.568256:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24383488, 24387584) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.568276:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24383488=0x1741000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.568336:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.568339:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24387584 00000020:00010000:2.0:1789760797.568347:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.568350:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.568353:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c8038bd8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.568356:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.568359:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.568367:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.568373:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.568840:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760797.569251:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 24383488<>24387584 00000100:00100000:2.0:1789760797.569296:0:2419:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:2.0:1789760797.569315:0:2419:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507399936:192.168.203.141@tcp:4:dd.0 00000100:00100000:2.0:1789760797.606933:0:2419:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973273 gen 3 00000008:00010000:2.0:1789760797.607003:0:2419:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000020:00200000:2.0:1789760797.607012:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760797.607023:0:2419:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760797.607044:0:2419:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000100:00100000:2.0:1789760797.607074:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507399936:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760797.607915:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5954, 5954] 00020000:00200000:2.0:1789760797.607925:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24387584 chunk: [24387584, 24391680) 24391680, 4096 00020000:00200000:2.0:1789760797.607930:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.607951:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5954, 5954] 00000008:00010000:2.0:1789760797.607970:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.607975:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.607988:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24387584, 24391680) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.608004:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24387584=0x1742000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.608078:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.608082:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24391680 00000020:00010000:2.0:1789760797.608090:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.608093:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.608097:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.608100:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.608103:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.608116:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.608124:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.608175:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac9c0 00000008:00100000:2.0:1789760797.608278:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021500 - object 0x280000401:3 offset 24387584<>24391680 00000080:00200000:2.0:1789760797.648160:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5954, 5954] 00000080:00200000:2.0:1789760797.648179:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.648185:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.648201:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24391680 i_blocks 49159 00000020:00200000:2.0:1789760797.648205:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.648215:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.648251:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24391680, count: 4096 00000080:00200000:2.0:1789760797.648258:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24391680, bytes: 4096 00000080:00200000:2.0:1789760797.648269:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.648280:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.648282:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24391680, 24395776) 0 0 00000080:00200000:2.0:1789760797.648287:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.648310:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5955, 5955] 00020000:00200000:2.0:1789760797.648320:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24391680 chunk: [24391680, 24395776) 24395776, 4096 00020000:00200000:2.0:1789760797.648324:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.648345:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5955, 5955] 00000008:00010000:2.0:1789760797.648364:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.648370:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.648418:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24391680, 24395776) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.648432:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24391680=0x1743000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.648490:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.648492:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24395776 00000020:00010000:2.0:1789760797.648500:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.648504:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.648507:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.648510:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.648512:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.648519:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.648525:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.648541:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760797.648657:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 24391680<>24395776 00000100:00100000:2.0:1789760797.693532:0:2418:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973275 gen 3 00000008:00010000:2.0:1789760797.693707:0:2418:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25c50acc00 00000020:00200000:2.0:1789760797.693714:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:2.0:1789760797.693725:0:2418:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:2.0:1789760797.693745:0:2418:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25c50acc00 00000100:00100000:2.0:1789760797.693773:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507400192:192.168.203.141@tcp:4:dd.0 00000080:00200000:1.0:1789760797.695070:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5955, 5955] 00000080:00200000:1.0:1789760797.695095:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:1.0:1789760797.695101:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:1.0:1789760797.695120:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24395776 i_blocks 49159 00000020:00200000:1.0:1789760797.695126:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:1.0:1789760797.695140:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:1.0:1789760797.695184:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24395776, count: 4096 00000080:00200000:1.0:1789760797.695193:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24395776, bytes: 4096 00000080:00200000:1.0:1789760797.695207:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:1.0:1789760797.695218:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:1.0:1789760797.695220:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24395776, 24399872) 0 0 00000080:00200000:1.0:1789760797.695225:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:1.0:1789760797.695243:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5956, 5956] 00020000:00200000:1.0:1789760797.695252:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24395776 chunk: [24395776, 24399872) 24399872, 4096 00020000:00200000:1.0:1789760797.695257:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:1.0:1789760797.695279:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5956, 5956] 00000008:00010000:1.0:1789760797.695298:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:1.0:1789760797.695304:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25d1e617b0, osc lock ffff8d25c57ef690, flags 0x0 00000080:00200000:1.0:1789760797.695317:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24395776, 24399872) append 0 append lockpos 0 00000080:00200000:1.0:1789760797.695332:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24395776=0x1744000, pages 1 (max 131072), lockless, parallel 00000020:00200000:1.0:1789760797.695425:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:1.0:1789760797.695428:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24399872 00000020:00010000:1.0:1789760797.695436:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:1.0:1789760797.695440:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:1.0:1789760797.695443:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d1e617b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:1.0:1789760797.695446:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:1.0:1789760797.695449:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d1e617b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:1.0:1789760797.695458:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:1.0:1789760797.695465:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:1.0:1789760797.695486:0:9279:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25d1451cc0 00000008:00100000:1.0:1789760797.695607:0:9279:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8169880 - object 0x280000401:3 offset 24395776<>24399872 00000100:00100000:0.0:1789760797.695722:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507400320:192.168.203.141@tcp:4:dd.0 00000100:00100000:1.0:1789760797.742636:0:2417:0:(client.c:3125:ptlrpc_free_committed()) lustre-OST0000-osc-ffff8d25c5405000: committing for last_committed 4294973276 gen 3 00000008:00010000:1.0:1789760797.742694:0:2417:0:(osc_cache.c:908:osc_extent_finish()) ### no dlm lock: extent: ffff8d25d1451cc0 00000020:00200000:1.0:1789760797.742703:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000020:00200000:1.0:1789760797.742714:0:2417:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000008:00010000:1.0:1789760797.742804:0:2417:0:(osc_cache.c:497:__osc_extent_remove()) ### no dlm lock: extent: ffff8d25d1451cc0 00000100:00100000:1.0:1789760797.742837:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507400320:192.168.203.141@tcp:4:dd.0 00000080:00200000:2.0:1789760797.743205:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5956, 5956] 00000080:00200000:2.0:1789760797.743221:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.743227:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.743244:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24399872 i_blocks 49159 00000020:00200000:2.0:1789760797.743248:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.743259:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.743293:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24399872, count: 4096 00000080:00200000:2.0:1789760797.743300:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24399872, bytes: 4096 00000080:00200000:2.0:1789760797.743313:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.743321:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.743323:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24399872, 24403968) 0 0 00000080:00200000:2.0:1789760797.743327:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.743343:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5957, 5957] 00020000:00200000:2.0:1789760797.743352:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24399872 chunk: [24399872, 24403968) 24403968, 4096 00020000:00200000:2.0:1789760797.743356:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.743407:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5957, 5957] 00000008:00010000:2.0:1789760797.743426:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.743432:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.743487:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24399872, 24403968) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.743502:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24399872=0x1745000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.743730:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.743734:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24403968 00000020:00010000:2.0:1789760797.743743:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.743747:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.743749:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.743754:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.743757:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.743765:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.743772:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.743794:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50acc00 00000008:00100000:2.0:1789760797.743910:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021c00 - object 0x280000401:3 offset 24399872<>24403968 00000080:00200000:2.0:1789760797.871407:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5957, 5957] 00000080:00200000:2.0:1789760797.871424:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.871430:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.871721:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24403968 i_blocks 49159 00000020:00200000:2.0:1789760797.871728:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.871739:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.871773:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24403968, count: 4096 00000080:00200000:2.0:1789760797.871781:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24403968, bytes: 4096 00000080:00200000:2.0:1789760797.871797:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.871807:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.871809:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24403968, 24408064) 0 0 00000080:00200000:2.0:1789760797.871814:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.871831:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5958, 5958] 00020000:00200000:2.0:1789760797.871840:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24403968 chunk: [24403968, 24408064) 24408064, 4096 00020000:00200000:2.0:1789760797.871845:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.871866:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5958, 5958] 00000008:00010000:2.0:1789760797.871884:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.871890:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.871901:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24403968, 24408064) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.871915:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24403968=0x1746000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.871975:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.871978:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24408064 00000020:00010000:2.0:1789760797.871985:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.871988:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.871992:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.871996:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.871998:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.872005:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.872207:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.872229:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac900 00000008:00100000:2.0:1789760797.872336:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8023480 - object 0x280000401:3 offset 24403968<>24408064 00000080:00200000:2.0:1789760797.943099:0:12700:0:(file.c:2205:ll_file_io_generic()) Range unlock [5958, 5958] 00000080:00200000:2.0:1789760797.943116:0:12700:0:(vvp_io.c:304:vvp_io_fini()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 need write layout 0, restore needed 0, invalidate_lock 0 00000080:00200000:2.0:1789760797.943122:0:12700:0:(file.c:2238:ll_file_io_generic()) tbf: 2 io complete with rc: 0, result: 4096, restart: 0 00000080:00200000:2.0:1789760797.943137:0:12700:0:(file.c:1775:ll_merge_attr_nolock()) [0x200000402:0x6:0x0] updating i_size 24408064 i_blocks 49159 00000020:00200000:2.0:1789760797.943141:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 0 items remaining. 00000080:00200000:2.0:1789760797.943161:0:12700:0:(file.c:2324:ll_file_io_generic()) iot: 2, result: 4096 00000080:00200008:2.0:1789760797.943196:0:12700:0:(file.c:2650:do_file_write_iter()) START file tbf:[0x200000402:0x6:0x0], ppos: 24408064, count: 4096 00000080:00200000:2.0:1789760797.943203:0:12700:0:(file.c:2090:ll_file_io_generic()) tbf: write ppos: 24408064, bytes: 4096 00000080:00200000:2.0:1789760797.943213:0:12700:0:(file.c:1848:ll_io_set_mirror()) tbf: desiginated mirror: 0 00000020:00200000:2.0:1789760797.943221:0:12700:0:(cl_io.c:214:cl_io_rw_init()) header@000000000afddd2a[0x110, 2, [0x200000402:0x6:0x0] hash] 00000020:00200000:2.0:1789760797.943224:0:12700:0:(cl_io.c:214:cl_io_rw_init()) io range: 2 [24408064, 24412160) 0 0 00000080:00200000:2.0:1789760797.943228:0:12700:0:(vvp_io.c:1826:vvp_io_init()) [0x200000402:0x6:0x0] ignore/verify layout 0/0, layout version 0 restore needed 0 00000080:00200000:2.0:1789760797.943246:0:12700:0:(file.c:2168:ll_file_io_generic()) Range lock [5959, 5959] 00020000:00200000:2.0:1789760797.943254:0:12700:0:(lov_io.c:1277:lov_io_rw_iter_init()) stripe: 24408064 chunk: [24408064, 24412160) 24412160, 4096 00020000:00200000:2.0:1789760797.943259:0:12700:0:(lov_io.c:1085:lov_io_iter_init()) component[0] flags 0x10 00000080:00200000:2.0:1789760797.943278:0:12700:0:(vvp_io.c:214:vvp_io_one_lock_index()) lock: 1 [5959, 5959] 00000008:00010000:2.0:1789760797.943295:0:12700:0:(lustre_osc.h:162:osc_enq2ldlm_flags()) flags: 10 00000008:00010000:2.0:1789760797.943300:0:12700:0:(osc_lock.c:1312:osc_lock_init()) ### lock ffff8d25c3f5d7b0, osc lock ffff8d25c5fa6000, flags 0x0 00000080:00200000:2.0:1789760797.943311:0:12700:0:(vvp_io.c:1292:vvp_io_write_start()) tbf: write [24408064, 24412160) append 0 append lockpos 0 00000080:00200000:2.0:1789760797.943325:0:12700:0:(rw26.c:362:ll_direct_IO()) VFS Op:inode=[0x200000402:0x6:0x0](ffff8d25e05e1148), size=4096 (max 536870912), offset=24408064=0x1747000, pages 1 (max 131072), lockless, parallel 00000020:00200000:2.0:1789760797.943424:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000080:00200000:2.0:1789760797.943427:0:12700:0:(rw26.c:544:ll_direct_IO()) result 0 tot_bytes 4096 count 4096 file_offset 24412160 00000020:00010000:2.0:1789760797.943436:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) release lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_release():245 00000020:00010000:2.0:1789760797.943439:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.943442:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) cancel lock: ffff8d25c3f5d7b0 (ffff8d25c80383f8/1) at cl_lock_cancel():142 00000020:00010000:2.0:1789760797.943447:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25d0be93f8 (ffff8d25c8038d88/0) at cl_lock_fini():72 00000020:00010000:2.0:1789760797.943450:0:12700:0:(cl_lock.c:36:__cl_lock_trace()) destroy lock: ffff8d25c3f5d7b0 (ffff8d25c8038d88/1) at cl_lock_fini():72 00000080:00200000:2.0:1789760797.943457:0:12700:0:(vvp_io.c:533:vvp_io_advance()) advancing 4096 bytes 00000020:00200000:2.0:1789760797.943464:0:12700:0:(cl_io.c:1849:__cl_sync_io_note()) Noting completion of 1 items, 1 items remaining. 00000008:00010000:2.0:1789760797.943478:0:416:0:(osc_cache.c:2020:try_to_add_extent_for_io()) ### no dlm lock: extent: ffff8d25c50ac180 00000008:00100000:2.0:1789760797.943600:0:416:0:(osc_request.c:2000:osc_brw_prep_request()) brw rpc ffff8d25c8021f80 - object 0x280000401:3 offset 24408064<>24412160 00010000:00010000:0.0:1789760798.523686:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760801.595700:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760804.667601:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760807.739593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760810.811546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760813.819704:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2418:1876699507403136:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760813.819790:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507403392:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760813.819830:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:e0668622-7dd7-45f4-a020-184f7e88cc06:2418:1876699507403648:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760813.819872:0:2418:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 3 async RPCs [1->0] 00000100:00100000:2.0:1789760813.819878:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507403264:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760813.819905:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507403520:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760813.819922:0:2418:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:e0668622-7dd7-45f4-a020-184f7e88cc06:2418:1876699507403776:192.168.203.141@tcp:400:kworker.0 00000100:00080000:0.0:1789760813.831305:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760813.831311:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760813.831314:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760813.831321:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 425 00000100:00080000:0.0:1789760813.831333:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760813.831343:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760813.831366:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507403904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1789760813.883571:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760814.854884:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760814.854890:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760814.854893:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760814.854898:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 280 00000100:00080000:0.0:1789760814.854910:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760814.854944:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507404032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789760814.855008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507404032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789760814.855048:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2418:1876699507403136:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760814.855059:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2418:1876699507401600:192.168.203.141@tcp:400:kworker.0 00000100:00100000:1.0:1789760814.855093:0:2417:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [0->1] 00000100:00100000:1.0:1789760814.855101:0:2417:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507404160:192.168.203.141@tcp:400:kworker.0 00000100:00080000:0.0:1789760814.858727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760814.858739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760814.858741:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760814.858746:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 280 00000100:00080000:0.0:1789760814.858757:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760814.858768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760814.858785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507404288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760814.858810:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760814.858815:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760814.858822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760814.858829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760814.858835:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760814.858846:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 345 00000100:00100000:1.0:1789760814.858852:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507404288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760814.858857:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760814.858869:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760814.858886:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507404416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760814.858948:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789760814.858953:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507404416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760814.858954:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760814.858960:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789760814.858962:0:2416:0:(ptlrpcd.c:344:ptlrpcd_check()) transfer 1 async RPCs [1->0] 00000100:00080000:0.0:1789760814.858970:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 280 00000100:00100000:1.0:1789760814.858977:0:2416:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_00:e0668622-7dd7-45f4-a020-184f7e88cc06:0:1876699507404544:192.168.203.141@tcp:400:kworker.0 00000100:00080000:0.0:1789760814.858980:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760814.858989:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760814.859008:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507404672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760814.859031:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760814.859037:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760814.859040:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789760814.859044:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507404672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760814.859046:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760814.859055:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760814.859059:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760814.859062:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760814.859069:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 345 00000100:00080000:0.0:1789760814.859077:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760814.859083:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760814.859094:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507404800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760814.859131:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507404800:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760816.955530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00000400:2.0:1789760819.963521:0:2418:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760803/real 1789760803] req@ffff8d25c8020700 x1876699507401984/t0(0) o400->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760819.976347:0:2418:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25c8020700 x1876699507401984/t0(0) o400->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760819.976362:0:2418:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8d25c8066800 already not connected (conn 4, was 3): CONNECTING 00000100:00000400:2.0:1789760819.976423:0:2418:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760803/real 1789760803] req@ffff8d25c8022d80 x1876699507402240/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760819.976438:0:2418:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25c8022d80 x1876699507402240/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760819.976446:0:2418:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c810b800 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:2.0:1789760819.976452:0:2418:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760803/real 1789760803] req@ffff8d25c8021180 x1876699507402112/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760819.976464:0:2418:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25c8021180 x1876699507402112/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760819.976481:0:2418:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c810f800 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:2.0:1789760819.976485:0:2418:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760803/real 1789760803] req@ffff8d25c8020000 x1876699507401856/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760819.976496:0:2418:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25c8020000 x1876699507401856/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760819.976503:0:2418:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c8061800 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:2.0:1789760819.976508:0:2418:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760803/real 1789760803] req@ffff8d25c8023100 x1876699507401728/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760819.976520:0:2418:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25c8023100 x1876699507401728/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760819 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760819.976527:0:2418:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c8065000 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1789760819.976545:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507401984:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760819.976551:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:e0668622-7dd7-45f4-a020-184f7e88cc06:2418:1876699507402240:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760819.976556:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:e0668622-7dd7-45f4-a020-184f7e88cc06:2418:1876699507402112:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760819.976561:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507401856:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760819.976566:0:2418:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_00:7fdb621d-3376-4597-a0ab-d89800273221:2418:1876699507401728:192.168.203.141@tcp:400:kworker.0 00010000:00010000:0.0:1789760820.027567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760823.099522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760823.995521:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760813/real 1789760813] req@ffff8d25d25bbb80 x1876699507403904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760823 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760823.995558:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507403904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760823 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760823.995580:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760823.995604:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760823.995608:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507403904:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1789760823.995661:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760823.995672:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760823.995676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760823.995679:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760823.995682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level CONNECTING/4 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760823.995686:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760823.995689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760823.995693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760823.995698:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760823.995708:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 796 00000100:00080000:0.0:1789760823.995719:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1789760823.995727:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760823.995739:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760823.995774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507404928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760823.995804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760823.995808:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760823.995811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760823.995816:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:2.0:1789760823.995852:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507404928:192.168.203.141@tcp:8:kworker.0 00000100:00000400:2.0:1789760824.059528:0:2419:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760808/real 1789760808] req@ffff8d25d1b0b480 x1876699507402624/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760824.068582:0:2419:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25d1b0b480 x1876699507402624/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760824.068593:0:2419:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c8061800 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:2.0:1789760824.068600:0:2419:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760808/real 1789760808] req@ffff8d25d1b08380 x1876699507402880/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760824.068612:0:2419:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25d1b08380 x1876699507402880/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760824.068634:0:2419:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c810f800 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:2.0:1789760824.068638:0:2419:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760808/real 1789760808] req@ffff8d25d1b09180 x1876699507402496/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760824.068646:0:2419:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25d1b09180 x1876699507402496/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760824.068652:0:2419:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c8065000 already not connected (conn 2, was 1): CONNECTING 00000100:00000400:2.0:1789760824.068655:0:2419:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760808/real 1789760808] req@ffff8d25d1b0aa00 x1876699507402752/t0(0) o400->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760824.068663:0:2419:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25d1b0aa00 x1876699507402752/t0(0) o400->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760824.068668:0:2419:0:(import.c:214:ptlrpc_set_import_discon()) osc: import ffff8d25c8066800 already not connected (conn 5, was 3): CONNECTING 00000100:00000400:2.0:1789760824.068673:0:2419:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760808/real 1789760808] req@ffff8d25d1b09c00 x1876699507403008/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760824.068699:0:2419:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25d1b09c00 x1876699507403008/t0(0) o400->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760824.068704:0:2419:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c810b800 already not connected (conn 2, was 1): CONNECTING 00000100:00100000:2.0:1789760824.068720:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507402624:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760824.068725:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2419:1876699507402880:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760824.068729:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507402496:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760824.068733:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:7fdb621d-3376-4597-a0ab-d89800273221:2419:1876699507402752:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760824.068736:0:2419:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_01_01:e0668622-7dd7-45f4-a020-184f7e88cc06:2419:1876699507403008:192.168.203.141@tcp:400:kworker.0 00000100:00100000:2.0:1789760825.019548:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760814/real 1789760814] req@ffff8d25d25ba680 x1876699507404800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019591:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507404800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019604:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760814/real 1789760814] req@ffff8d25d25b9180 x1876699507404672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019616:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507404672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019626:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760814/real 1789760814] req@ffff8d25d25b8380 x1876699507404416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019638:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507404416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019649:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760814/real 1789760814] req@ffff8d25d25bb800 x1876699507404288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019664:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507404288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019672:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760814/real 1789760814] req@ffff8d25d25b9c00 x1876699507404032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760825.019681:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507404032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760824 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760825.019700:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760825.019723:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760825.019727:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507404800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760825.019732:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760825.019736:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760825.019739:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507404672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760825.019742:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760825.019744:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760825.019747:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507404416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760825.019749:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760825.019752:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760825.019754:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507404288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760825.019757:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760825.019759:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760825.019761:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507404032:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789760825.019818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760825.019826:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760825.019830:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760825.019839:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 797 00000100:00080000:0.0:1789760825.019849:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1789760825.019855:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760825.019887:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507405056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760825.019917:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760825.019920:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760825.019922:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760825.019925:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 797 00000100:00080000:0.0:1789760825.019927:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1789760825.019931:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760825.019940:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760825.019952:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507405184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760825.019959:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507405056:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789760825.019963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760825.019967:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760825.019968:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760825.019972:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 797 00000100:00080000:0.0:1789760825.019974:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1789760825.019976:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760825.019980:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760825.019988:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507405312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760825.019996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760825.019999:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760825.020002:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760825.020007:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760825.020008:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760825.020011:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 797 00000100:00080000:0.0:1789760825.020014:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1789760825.020016:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760825.020020:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760825.020026:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507405440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760825.020033:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760825.020036:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760825.020037:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760825.020041:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 797 00000100:00080000:0.0:1789760825.020044:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 5s 00000100:00080000:0.0:1789760825.020046:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760825.020049:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760825.020060:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507405568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760825.020088:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760825.020123:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760825.020157:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507405440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760825.020186:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507405568:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760826.171554:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760829.051670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760829.051684:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760829.051690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760829.051695:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760829.051700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760829.051705:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760829.051709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760829.051715:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760829.051721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760829.051726:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760829.051731:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760829.051736:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1789760829.243596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760830.075502:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760830.075514:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760830.075519:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760830.075524:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760830.075529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760830.075534:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760830.075538:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760830.075543:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760830.075547:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760830.075551:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760830.075554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760830.075559:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00000400:1.0:1789760830.203487:0:2417:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760814/real 1789760814] req@ffff8d25d25bb480 x1876699507404160/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760830 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789760830.203526:0:2417:0:(client.c:2547:ptlrpc_expire_one_request()) @@@ TIMEOUT-NORESEND: req@ffff8d25d25bb480 x1876699507404160/t0(0) o400->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 224/224 e 0 to 1 dl 1789760830 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789760830.203535:0:2417:0:(import.c:214:ptlrpc_set_import_discon()) mdc: import ffff8d25c8065000 already not connected (conn 3, was 1): CONNECTING 00000100:00100000:1.0:1789760830.203551:0:2417:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_00_01:7fdb621d-3376-4597-a0ab-d89800273221:2417:1876699507404160:192.168.203.141@tcp:400:kworker.0 00010000:00010000:0.0:1789760832.315549:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760835.323554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760835.323566:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760835.323571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760835.323580:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760835.323584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760835.323590:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760835.323595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760835.323599:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760835.323605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760835.323609:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760835.323613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760835.323617:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1789760835.387590:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760838.267464:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760823/real 1789760823] req@ffff8d25d25b9f80 x1876699507404928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760838 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760838.267504:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507404928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760838 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760838.267543:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760838.267567:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760838.267572:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507404928:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1789760838.267709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.267721:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760838.267725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.267729:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760838.267732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.267735:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760838.267738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.267742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760838.267746:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760838.267759:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 806 00000100:00080000:2.0:1789760838.267775:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 10s 00000100:00080000:2.0:1789760838.267785:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760838.267800:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789760838.267849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507405696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760838.267888:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.267892:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760838.267896:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760838.267900:0:37:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1789760838.267943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405696:192.168.203.141@tcp:8:kworker.0 00010000:00010000:0.0:1789760838.459613:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000800:00000100:0.0:1789760838.651564:0:2407:0:(socklnd_cb.c:2254:ksocknal_find_timed_out_conn()) A connection with 12345-192.168.203.141@tcp (192.168.203.141:988) timed out; the network or node may be down. 00000100:00100000:0.0:1789760840.315474:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760825/real 1789760825] req@ffff8d25d25bad80 x1876699507405184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315515:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507405184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315541:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760825/real 1789760825] req@ffff8d25d25b8700 x1876699507405312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315550:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507405312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315557:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760825/real 1789760825] req@ffff8d25d25bb100 x1876699507405440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315577:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507405440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315583:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760825/real 1789760825] req@ffff8d25d25b9c00 x1876699507405568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315591:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507405568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315607:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760825/real 1789760825] req@ffff8d25d25b8e00 x1876699507405056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760840.315617:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507405056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760840 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760840.315648:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760840.315672:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760840.315675:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405184:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760840.315680:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760840.315686:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760840.315688:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760840.315692:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760840.315695:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760840.315697:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507405440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760840.315699:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760840.315702:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760840.315704:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507405568:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760840.315709:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760840.315713:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760840.315715:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507405056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760840.315957:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507405824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760840.316113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760840.316158:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507406080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760840.316180:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760840.316204:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406336:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760841.531572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760843.323564:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760843.323573:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760843.323576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760843.323581:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760843.323585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760843.323587:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760843.323591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760843.323594:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760843.323599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760843.323602:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760843.323605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760843.323609:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00010000:00010000:0.0:1789760844.603671:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760847.675648:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789760850.491771:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.203.141:988 00000400:00000100:2.0:1789760850.491848:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789760850.491868:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760850.491930:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760850.491947:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760850.491963:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760850.491966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760840/real 1789760850] req@ffff8d25d1b0ad80 x1876699507405952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789760850.491980:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760850.492026:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507405952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760850.492041:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760850.492067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.492084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760850.492096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760840/real 1789760850] req@ffff8d25d1b0b100 x1876699507406080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760850.492102:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.492116:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1789760850.492123:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507406080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760850.492124:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.492147:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760850.492154:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760850.492164:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 823 00000100:00080000:0.0:1789760850.492188:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760850.492197:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 15s 00000100:00080000:0.0:1789760850.492231:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789760850.492233:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760850.492245:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507406080:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789760850.492256:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760850.492259:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760840/real 1789760850] req@ffff8d25d1b0bb80 x1876699507406208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760850.492284:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507406208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760850.492296:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789760850.492305:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507406464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760850.492308:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.492324:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760850.492344:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760840/real 1789760850] req@ffff8d25d1b08e00 x1876699507405824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760850.492348:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.492366:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760850.492373:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760850.492380:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 823 00000100:00080000:2.0:1789760850.492387:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 15s 00000100:00100000:0.0:1789760850.492427:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507405824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760850.492440:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760850.492453:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760850.492463:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789760850.492464:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.492476:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507405824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789760850.492493:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507406592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760850.492511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.492537:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760850.492546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.492552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760850.492555:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760850.492563:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 823 00000100:00080000:2.0:1789760850.492568:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789760850.492574:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760850.492579:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507406464:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789760850.492584:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760850.492606:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507406720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760850.492627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.492638:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000800:00000100:0.0:1789760850.492652:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789760850.492655:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.492666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760850.492671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760850.492681:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 823 00000100:00080000:2.0:1789760850.492691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 15s 00000100:00100000:0.0:1789760850.492698:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760850/real 1789760850] req@ffff8d25c8020000 x1876699507406464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760850.492716:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760850.492746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507406848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760850.492753:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507406464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760850.492792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760850.492795:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760850.492806:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:0.0:1789760850.492807:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789760850.492819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760850.492821:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507406464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760850.492833:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00100000:0.0:1789760850.492842:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507406592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760850.492870:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760850.492890:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760850/real 1789760850] req@ffff8d25c8021180 x1876699507406592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760850.492903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507406592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760850.492912:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760850.492916:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.492919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507406592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760850.492926:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760840/real 1789760850] req@ffff8d25d1b08700 x1876699507406336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760850.492937:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507406336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760850.492947:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760850.492975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.492979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760850.493000:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760850.493006:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level CONNECTING/4 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760850.493013:0:36:0:(pinger.c:247:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: not pinging (in recovery or recovery disabled: CONNECTING) 00000100:00080000:2.0:1789760850.493020:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 1 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789760850.493031:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789760850.493041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760850.493045:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789760850.493055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760850/real 1789760850] req@ffff8d25c8022d80 x1876699507406720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760850.493055:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 823 00000100:00080000:2.0:1789760850.493071:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 15s 00000100:00080000:2.0:1789760850.493082:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760850.493087:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507406720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789760850.493093:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789760850.493119:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760850.493123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.493130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760850.493137:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507406976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760850.493147:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507406848:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1789760850.493212:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760850.493249:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760850/real 1789760850] req@ffff8d25c8020700 x1876699507406848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760850.493265:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507406848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760850.493275:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760850.493279:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.493284:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507406848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760850.493303:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760850.493328:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760850.493341:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760850/real 1789760850] req@ffff8d25c8023b80 x1876699507406976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760850.493356:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507406976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760875 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760850.493365:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760850.493369:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760850.493372:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507406976:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760850.747635:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760853.819644:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760855.803691:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507407104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760855.803833:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507407232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760855.803891:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507407360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760855.803941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507407488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760855.803994:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507407616:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760856.891593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760858.875536:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has timed out for slow reply: [sent 1789760838/real 1789760838] req@ffff8d25c8023100 x1876699507405696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760858 ref 1 fl Rpc:XNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760858.875579:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507405696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760858 ref 1 fl Rpc:XNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760858.875603:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760858.875629:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760858.875637:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507405696:192.168.203.141@tcp:8:kworker.0 00000400:00000100:1.0:1789760858.875687:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.203.141:988 00000400:00000100:1.0:1789760858.875760:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760858.875783:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760858.875830:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760858.875859:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760858.875871:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760855/real 1789760858] req@ffff8d25d1b09f80 x1876699507407232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1789760858.875894:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760858.875956:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09f80 x1876699507407232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760858.875972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760858.875979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000800:00000100:1.0:1789760858.875986:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760858.875992:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507407232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760858.876006:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760855/real 1789760858] req@ffff8d25d1b0a680 x1876699507407360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760858.876020:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507407360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760858.876028:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760858.876030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760858.876034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507407360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760858.876040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760855/real 1789760858] req@ffff8d25d1b09880 x1876699507407104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760858.876052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507407104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760858.876059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760858.876062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760858.876065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507407104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760858.876102:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507407744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760858.876207:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760858.876252:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760858/real 1789760858] req@ffff8d25d1b08e00 x1876699507407744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760883 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760858.876271:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507407744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760883 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760858.876283:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760858.876288:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760858.876293:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507407744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760858.876303:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760855/real 1789760858] req@ffff8d25d1b08700 x1876699507407616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760858.876321:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507407616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760858.876332:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760858.876337:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760858.876343:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507407616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760858.876353:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760855/real 1789760858] req@ffff8d25d1b0b800 x1876699507407488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760858.876371:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507407488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760880 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760858.876413:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760858.876419:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760858.876427:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507407488:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760859.963542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760860.923783:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507407872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760860.923945:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760860.923999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760860.924035:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760860.924066:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507408384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760860.924116:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507408512:192.168.203.141@tcp:38:kworker.0 00000400:00000100:1.0:1789760861.947894:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.203.141:988 00000400:00000100:1.0:1789760861.947950:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760861.947975:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760861.948056:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760861.948107:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760861.948126:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760861.948150:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760861.948177:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760861.948785:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760860/real 1789760861] req@ffff8d25d1b09180 x1876699507408512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760861.948836:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507408512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760861.948882:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760861.948891:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760861.948900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507408512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760861.948937:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760860/real 1789760861] req@ffff8d25d1b0b100 x1876699507408000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760861.948964:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507408000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760861.948981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760861.948989:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760861.948996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760861.949009:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760860/real 1789760861] req@ffff8d25d1b0ad80 x1876699507408128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760861.949053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507408128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760861.949071:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760861.949080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760861.949107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760861.949123:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760860/real 1789760861] req@ffff8d25d1b09c00 x1876699507408256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760861.949160:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507408256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760861.949180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760861.949188:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760861.949195:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760861.949208:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760860/real 1789760861] req@ffff8d25d1b0aa00 x1876699507408384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760861.949232:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507408384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760861.949249:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760861.949256:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760861.949263:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507408384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760861.949276:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760860/real 1789760861] req@ffff8d25d1b0bb80 x1876699507407872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760861.949294:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507407872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760885 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760861.949304:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760861.949308:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760861.949312:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507407872:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789760863.035556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760866.043781:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507408640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760866.043936:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760866.044008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760866.044113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760866.044171:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507409152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760866.044205:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507409280:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760866.107579:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789760869.115753:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.203.141:988 00000400:00000100:1.0:1789760869.115828:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789760869.115854:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760869.115899:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760869.115910:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760869.115919:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760869.115927:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789760869.115934:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760869.116006:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760866/real 1789760869] req@ffff8d25d1b08a80 x1876699507409280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760869.116075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507409280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760869.116095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760869.116103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760869.116111:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507409280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760869.116124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760866/real 1789760869] req@ffff8d25d1b09500 x1876699507409024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760869.116147:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507409024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760869.116162:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760869.116168:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760869.116174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760869.116186:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760866/real 1789760869] req@ffff8d25d1b08000 x1876699507409152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760869.116215:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507409152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760869.116238:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760869.116244:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760869.116249:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507409152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760869.116260:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760866/real 1789760869] req@ffff8d25d1b0b480 x1876699507408768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760869.116290:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507408768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760869.116302:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760869.116308:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760869.116313:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760869.116324:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760866/real 1789760869] req@ffff8d25d1b0a300 x1876699507408896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760869.116345:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507408896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760869.116359:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760869.116365:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760869.116371:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507408896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760869.116445:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760866/real 1789760869] req@ffff8d25d1b08380 x1876699507408640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760869.116471:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507408640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760869.116484:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760869.116489:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760869.116495:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507408640:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789760869.179650:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760871.163844:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507409408:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1789760871.163954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760871.164057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760871] req@ffff8d25d1b08380 x1876699507409408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760871.164118:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507409408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760871.164148:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760871.164164:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760871.164174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507409408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760871.164191:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760871.164242:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760871.164270:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760871] req@ffff8d25d1b0a300 x1876699507409536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760871.164296:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507409536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760871.164323:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760871.164330:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760871.164337:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760871.164349:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760871.164484:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760871.164524:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760871] req@ffff8d25d1b0b480 x1876699507409664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760871.164553:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507409664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760871.164569:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760871.164585:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760871.164593:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760871.164645:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760871.164692:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760871.164720:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760871] req@ffff8d25d1b08000 x1876699507409792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760871.164744:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507409792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760871.164762:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760871.164768:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760871.164776:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507409792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760871.164788:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507409920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760871.164839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760871.164867:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760871] req@ffff8d25d1b09500 x1876699507409920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760871.164890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507409920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760871.164906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760871.164913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760871.164920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507409920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760871.164932:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507410048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760871.164972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760871.164994:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760871/real 1789760871] req@ffff8d25d1b08a80 x1876699507410048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760871.165041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507410048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760871.165059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760871.165066:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760871.165091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507410048:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760872.251508:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760875.323478:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760876.283656:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507410176:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760876.283771:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507410304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760876.283869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507410432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760876.283903:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507410560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760876.283935:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507410688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760876.283952:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507410816:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760878.395537:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789760879.355634:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.203.141:988 00000400:00000100:0.0:1789760879.355670:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789760879.355691:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789760879.355731:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789760879.355745:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789760879.355764:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffff8d25d1b0bb80 x1876699507410176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1789760879.355767:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789760879.355782:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789760879.355797:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507410176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1789760879.355799:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00080000:2.0:1789760879.355815:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.355821:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760879.355826:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507410176:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789760879.355834:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffff8d25d1b0aa00 x1876699507410304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.355848:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507410304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.355856:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.355860:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760879.355864:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507410304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760879.355886:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffff8d25d1b09180 x1876699507410816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.355900:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507410816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.355916:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.355921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760879.355926:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507410816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760879.355934:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffff8d25d1b09c00 x1876699507410432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.355949:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507410432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.355975:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.355979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760879.355984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507410432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760879.356005:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffff8d25d1b0ad80 x1876699507410560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.356021:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507410560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.356031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.356035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760879.356039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507410560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789760879.356055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760876/real 1789760879] req@ffff8d25d1b0b100 x1876699507410688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760879.356078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507410688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760879.356087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760879.356091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760879.356096:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507410688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789760881.403500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760881.403511:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760881.403516:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760881.403528:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 859 00000100:00080000:0.0:1789760881.403542:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760881.403553:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760881.403603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507410944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760881.403652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760881.403659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760881.403661:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760881.403669:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 859 00000100:00080000:0.0:1789760881.403675:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760881.403682:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760881.403696:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760881.403697:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507410944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760881.403747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507411072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760881.403775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760881.403778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760881.403780:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760881.403783:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 859 00000100:00080000:0.0:1789760881.403786:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760881.403790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789760881.403796:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789760881.403797:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760881.403814:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507411200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760881.403824:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760881.403827:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760881.403829:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760881.403834:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 859 00000100:00080000:0.0:1789760881.403840:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789760881.403846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760881/real 1789760881] req@ffff8d25d25b8e00 x1876699507410944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760881.403848:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760881.403857:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760881.403878:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507411328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760881.403880:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507410944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760881.403908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760881.403913:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760881.403914:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789760881.403921:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789760881.403923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507410944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789760881.403925:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760881.403936:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 859 00000100:00080000:0.0:1789760881.403955:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760881.403960:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760881.403965:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760881.403971:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760881.403983:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507411456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760881.403996:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760881.404001:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:2.0:1789760881.404005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789760881.404006:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760881.404015:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 859 00000100:00080000:0.0:1789760881.404021:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760881.404026:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760881.404030:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760881/real 1789760881] req@ffff8d25d25b9c00 x1876699507411072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789760881.404034:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760881.404059:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507411584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760881.404061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507411072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760881.404092:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760881.404101:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760881.404106:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760881.404114:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760881.404139:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760881.404154:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760881/real 1789760881] req@ffff8d25d25bb100 x1876699507411200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760881.404167:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507411200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760881.404176:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760881.404179:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760881.404183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760881.404190:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789760881.404213:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760881.404227:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760881/real 1789760881] req@ffff8d25d25b8700 x1876699507411328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760881.404240:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507411328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760881.404249:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760881.404253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760881.404256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789760881.404275:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507411456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760881.404298:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760881.404315:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760881/real 1789760881] req@ffff8d25d25bad80 x1876699507411456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760881.404330:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507411456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760881.404339:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760881.404343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760881.404347:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507411456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760881.404354:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507411584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760881.404374:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760881.404427:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760881/real 1789760881] req@ffff8d25d25b9f80 x1876699507411584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760881.404441:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507411584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760881.404450:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760881.404454:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760881.404457:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507411584:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760881.467568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760884.539545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760886.523583:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507411712:192.168.203.141@tcp:250:kworker.0 00000800:00000100:2.0:1789760886.523659:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760886.523704:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760886/real 1789760886] req@ffff8d25d1b0b800 x1876699507411712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760886.523732:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507411712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760886.523749:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760886.523754:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760886.523758:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507411712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789760886.523779:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760886.523797:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760886.523805:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760886/real 1789760886] req@ffff8d25d1b08700 x1876699507411840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760886.523814:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507411840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760886.523820:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760886.523822:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760886.523824:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760886.523828:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760886.523838:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760886.523844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760886/real 1789760886] req@ffff8d25d1b08e00 x1876699507411968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760886.523851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507411968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760886.523857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760886.523859:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760886.523861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507411968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760886.523864:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789760886.523874:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760886.523881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760886/real 1789760886] req@ffff8d25d1b09880 x1876699507412096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760886.523890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507412096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760886.523911:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760886.523916:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760886.523919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789760886.523926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507412224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760886.523947:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760886.523958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760886/real 1789760886] req@ffff8d25d1b0a680 x1876699507412224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760886.523967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507412224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760886.523972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760886.523975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760886.523977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507412224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760886.523980:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507412352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760886.523993:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760886.524000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760886/real 1789760886] req@ffff8d25d1b09f80 x1876699507412352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760886.524007:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09f80 x1876699507412352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760886.524013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760886.524015:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760886.524017:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507412352:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760887.611545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760890.683559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760891.643475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760891.643486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760891.643490:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760891.643501:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 869 00000100:00080000:0.0:1789760891.643512:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760891.643521:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760891.643563:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507412480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760891.643590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760891.643593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760891.643595:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760891.643598:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 869 00000100:00080000:0.0:1789760891.643600:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760891.643603:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760891.643613:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760891.643622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507412608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760891.643631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760891.643634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760891.643636:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760891.643643:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 869 00000100:00080000:0.0:1789760891.643646:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789760891.643648:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507412480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789760891.643649:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760891.643659:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760891.643669:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507412736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760891.643676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760891.643679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760891.643680:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760891.643683:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 869 00000100:00080000:0.0:1789760891.643685:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760891.643689:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760891.643692:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760891.643701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507412864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760891.643709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760891.643712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760891.643713:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760891.643716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 869 00000100:00080000:0.0:1789760891.643719:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789760891.643725:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789760891.643730:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760891.643738:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760891.643748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507412992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760891.643757:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760891.643760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760891.643761:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760891.643766:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 869 00000100:00080000:0.0:1789760891.643771:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760891.643775:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760891.643779:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760891.643780:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760891/real 1789760891] req@ffff8d25d25bb800 x1876699507412480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760891.643793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507413120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789760891.643806:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507412480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760891.643817:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760891.643821:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760891.643824:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507412480:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789760891.643846:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760891.643877:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760891.643890:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760891/real 1789760891] req@ffff8d25d25b8380 x1876699507412608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760891.643898:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507412608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760891.643910:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760891.643913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760891.643915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760891.643921:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760891.643936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760891.643944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760891/real 1789760891] req@ffff8d25d25b9180 x1876699507412736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760891.643952:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507412736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760891.643957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760891.643960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760891.643962:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760891.643966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789760891.643976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760891.643982:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760891/real 1789760891] req@ffff8d25d25ba680 x1876699507412864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760891.643990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507412864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760891.643996:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760891.643997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760891.643999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507412864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789760891.644004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507412992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760891.644014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760891.644020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760891/real 1789760891] req@ffff8d25d25bbb80 x1876699507412992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760891.644028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507412992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760891.644033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760891.644035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760891.644037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507412992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760891.644040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507413120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789760891.644051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789760891.644057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760891/real 1789760891] req@ffff8d25d25b8a80 x1876699507413120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760891.644065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507413120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760891.644070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789760891.644071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789760891.644073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507413120:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760893.756503:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760894.779574:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760895.803622:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:2.0:1789760896.763618:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507413248:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789760896.763713:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507413376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760896.763746:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507413504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760896.763776:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507413632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789760896.763797:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507413760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789760896.763812:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507413888:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760896.827734:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760897.852498:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760898.875546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789760899.835779:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1022 -> 192.168.203.141:988 00000400:00000100:2.0:1789760899.835866:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789760899.835883:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760899.835916:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760899.835923:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760899.835928:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760899.835933:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760899.835940:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760899.835965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760896/real 1789760899] req@ffff8d25d1b08700 x1876699507413760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760899.835999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507413760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760899.836012:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760899.836017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760899.836022:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507413760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760899.836030:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760896/real 1789760899] req@ffff8d25d1b0b800 x1876699507413888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760899.836044:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507413888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760899.836053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760899.836055:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760899.836058:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507413888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760899.836063:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760896/real 1789760899] req@ffff8d25d1b0a680 x1876699507413376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760899.836071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507413376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760899.836077:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760899.836081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760899.836084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507413376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760899.836091:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760896/real 1789760899] req@ffff8d25d1b09880 x1876699507413504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760899.836104:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507413504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760899.836112:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760899.836115:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760899.836119:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507413504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760899.836126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760896/real 1789760899] req@ffff8d25d1b08e00 x1876699507413632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760899.836137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507413632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760899.836145:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760899.836148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760899.836151:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507413632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760899.836158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760896/real 1789760899] req@ffff8d25d1b09f80 x1876699507413248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760899.836169:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09f80 x1876699507413248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760921 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760899.836176:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760899.836178:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760899.836181:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507413248:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789760899.899567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760900.923533:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760901.883636:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507414016:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1789760901.883728:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760901.883808:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760901/real 1789760901] req@ffff8d25d1b0b100 x1876699507414016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760901.883842:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507414016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760901.883875:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760901.883884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760901.883894:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507414016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760901.883931:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507414144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760901.883972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760901.883991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760901/real 1789760901] req@ffff8d25d1b0ad80 x1876699507414144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760901.884006:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507414144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760901.884014:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760901.884017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760901.884021:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507414144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760901.884028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507414272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760901.884049:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760901.884060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760901/real 1789760901] req@ffff8d25d1b09c00 x1876699507414272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760901.884070:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507414272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760901.884078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760901.884081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760901.884084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507414272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760901.884090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507414400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760901.884108:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760901.884120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760901/real 1789760901] req@ffff8d25d1b09180 x1876699507414400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760901.884137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507414400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760901.884146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760901.884148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760901.884152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507414400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760901.884158:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507414528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760901.884178:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760901.884190:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760901/real 1789760901] req@ffff8d25d1b0aa00 x1876699507414528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760901.884202:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507414528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760901.884209:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760901.884213:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760901.884216:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507414528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760901.884240:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507414656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760901.884263:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760901.884275:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760901/real 1789760901] req@ffff8d25d1b0bb80 x1876699507414656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760901.884288:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507414656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760926 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760901.884302:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760901.884306:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760901.884309:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507414656:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789760901.947606:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760902.971513:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760903.995714:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760906.043533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760907.003579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760907.003610:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760907.003614:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760907.003624:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 884 00000100:00080000:2.0:1789760907.003633:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760907.003640:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760907.003701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507414784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760907.003731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760907.003736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760907.003739:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760907.003744:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 884 00000100:00080000:2.0:1789760907.003766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760907.003774:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760907.003808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760907.003825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507414912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760907.003838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760907.003840:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760907.003842:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760907.003846:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 884 00000100:00080000:2.0:1789760907.003851:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760907.003854:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760907.003859:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760907.003868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507415040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760907.003876:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760907.003881:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760907.003884:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760907.003894:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 884 00000100:00080000:2.0:1789760907.003898:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760907.003901:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760907.003905:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789760907.003915:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507415168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760907.003924:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760907.003927:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760907.003930:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760907.003933:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 884 00000100:00080000:2.0:1789760907.003937:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760907.003943:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760907.003947:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760907.003956:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507415296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760907.003963:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760907.003966:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760907.003967:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760907.003970:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 884 00000100:00080000:2.0:1789760907.003972:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760907.003975:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760907.003977:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760907.003982:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507415424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:2.0:1789760907.067618:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760909.115536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760910.139572:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760912.123475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.123483:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760912.123489:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.123498:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 889 00000100:00080000:2.0:1789760912.123507:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.123513:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760912.123546:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507415552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760912.123568:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.123571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760912.123573:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.123576:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 889 00000100:00080000:2.0:1789760912.123579:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.123582:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760912.123589:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760912.123599:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507415680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760912.123611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.123615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760912.123621:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.123628:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 889 00000100:00080000:2.0:1789760912.123631:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.123633:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760912.123636:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760912.123644:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507415808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760912.123655:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.123657:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760912.123659:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.123662:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 889 00000100:00080000:2.0:1789760912.123664:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.123666:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760912.123669:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789760912.123674:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507415936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760912.123682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.123685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760912.123686:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.123688:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 889 00000100:00080000:2.0:1789760912.123691:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.123694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760912.123696:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760912.123704:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507416064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760912.123724:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760912.123735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760912.123741:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760912.123745:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 889 00000100:00080000:2.0:1789760912.123750:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760912.123752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760912.123756:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760912.123765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507416192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1789760912.187549:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760913.211557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760915.259528:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760916.283555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760917.243503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760917.243514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760917.243518:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760917.243527:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 894 00000100:00080000:0.0:1789760917.243538:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760917.243546:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760917.243582:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507416320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760917.243611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760917.243615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760917.243617:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760917.243621:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 894 00000100:00080000:0.0:1789760917.243625:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760917.243630:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760917.243640:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760917.243652:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507416448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760917.243699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760917.243710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760917.243723:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760917.243741:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 894 00000100:00080000:0.0:1789760917.243747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760917.243754:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760917.243763:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760917.243781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507416576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760917.243791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760917.243795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760917.243797:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760917.243802:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 894 00000100:00080000:0.0:1789760917.243807:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760917.243810:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760917.243814:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760917.243825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507416704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760917.243851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760917.243854:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760917.243857:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760917.243864:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 894 00000100:00080000:0.0:1789760917.243879:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760917.243883:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760917.243888:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760917.243912:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507416832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760917.243990:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760917.243994:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760917.243997:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760917.244002:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 894 00000100:00080000:0.0:1789760917.244009:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760917.244025:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760917.244046:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760917.244063:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507416960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00010000:00010000:0.0:1789760918.331596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789760919.355610:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760921.403555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760922.363645:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507417088:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1789760922.363720:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760922.363796:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffff8d25d1b08a80 x1876699507417088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760922.363856:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507417088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760922.363869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760922.363873:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760922.363877:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507417088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760922.363899:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760922.363935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760922.363944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffff8d25d1b09500 x1876699507417216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760922.363953:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507417216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760922.363959:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760922.363962:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760922.363964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760922.363968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760922.363980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760922.363987:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffff8d25d1b08000 x1876699507417344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760922.363994:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507417344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760922.364000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760922.364002:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760922.364004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760922.364016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760922.364047:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760922.364054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffff8d25d1b0b480 x1876699507417472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760922.364062:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507417472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760922.364068:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760922.364070:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760922.364072:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760922.364076:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507417600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760922.364086:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760922.364092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffff8d25d1b0a300 x1876699507417600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760922.364099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507417600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760922.364104:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760922.364106:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760922.364108:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507417600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760922.364111:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507417728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760922.364121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760922.364127:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760922/real 1789760922] req@ffff8d25d1b08380 x1876699507417728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760922.364140:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507417728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760922.364145:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760922.364147:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760922.364149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507417728:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760924.475597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760927.483680:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507417856:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1789760927.483739:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760927.483777:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760927/real 1789760927] req@ffff8d25d1b08380 x1876699507417856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760927.483803:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507417856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760927.483821:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760927.483826:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760927.483832:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507417856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760927.483859:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760927.483877:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760927.483886:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760927/real 1789760927] req@ffff8d25d1b0a300 x1876699507417984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760927.483894:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507417984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760927.483904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760927.483906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760927.483909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507417984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760927.483912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760927.483923:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760927.483928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760927/real 1789760927] req@ffff8d25d1b0b480 x1876699507418112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760927.483936:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507418112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760927.483941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760927.483942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760927.483944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760927.483948:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760927.483958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760927.483964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760927/real 1789760927] req@ffff8d25d1b08000 x1876699507418240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760927.483971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507418240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760927.483976:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760927.483978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760927.483980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760927.483992:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507418368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760927.484004:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760927.484011:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760927/real 1789760927] req@ffff8d25d1b09500 x1876699507418368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760927.484018:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507418368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760927.484024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760927.484025:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760927.484027:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507418368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760927.484031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507418496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760927.484045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760927.484056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760927/real 1789760927] req@ffff8d25d1b08a80 x1876699507418496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760927.484068:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507418496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760927.484076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760927.484079:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760927.484082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507418496:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760927.547583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760930.619509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00100000:0.0:1789760932.603638:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507418624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789760932.603761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507419136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760932.603774:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760932.603778:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418752:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789760932.603779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760932.603784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760932.603794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 910 00000100:00080000:3.0:1789760932.603798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760932.603802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760932.603808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760932.603821:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507419264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760932.603851:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760932.603898:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760932.603960:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507419136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760932.604014:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507419264:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760932.667614:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760933.691522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760934.715500:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789760935.675865:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1789760935.675946:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789760935.675965:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760935.676006:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760935.676017:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760935.676029:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760935.676039:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789760935.676051:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789760935.676063:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760932/real 1789760935] req@ffff8d25d1b0ad80 x1876699507419136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760935.676096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507419136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760935.676109:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760935.676114:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760935.676119:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507419136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760935.676128:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760932/real 1789760935] req@ffff8d25d1b0b100 x1876699507419264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760935.676141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507419264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760935.676149:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760935.676152:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760935.676157:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507419264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760935.676164:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760932/real 1789760935] req@ffff8d25d1b0aa00 x1876699507418752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760935.676177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507418752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760935.676185:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760935.676189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760935.676193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760935.676200:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760932/real 1789760935] req@ffff8d25d1b09180 x1876699507418880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760935.676213:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507418880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760935.676222:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760935.676226:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760935.676229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507418880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760935.676234:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760932/real 1789760935] req@ffff8d25d1b09c00 x1876699507419008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760935.676244:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507419008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760935.676249:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760935.676251:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760935.676255:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760935.676260:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760932/real 1789760935] req@ffff8d25d1b0bb80 x1876699507418624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760935.676269:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507418624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760935.676276:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760935.676278:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760935.676282:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507418624:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1789760935.739578:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760936.763558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760937.723437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760937.723445:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760937.723449:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760937.723458:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 915 00000100:00080000:3.0:1789760937.723466:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760937.723472:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760937.723501:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09f80 x1876699507419392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760937.723535:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760937.723538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760937.723540:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760937.723544:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 915 00000100:00080000:3.0:1789760937.723548:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760937.723551:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760937.723559:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760937.723569:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507419520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760937.723579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760937.723583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760937.723585:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760937.723591:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 915 00000100:00080000:3.0:1789760937.723595:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760937.723600:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507419392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789760937.723602:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760937.723612:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760937.723628:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507419648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760937.723648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760937.723650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760937.723653:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760937.723656:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 915 00000100:00080000:3.0:1789760937.723661:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789760937.723668:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789760937.723668:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760937.723678:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789760937.723693:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507419776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760937.723705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760937.723709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760937.723712:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760937.723717:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 915 00000100:00100000:0.0:1789760937.723719:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760937/real 1789760937] req@ffff8d25d1b09f80 x1876699507419392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760937.723722:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760937.723730:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760937.723740:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760937.723752:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09f80 x1876699507419392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760937.723757:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507419904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760937.723770:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760937.723775:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760937.723780:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789760937.723782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760937.723787:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789760937.723791:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507419392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789760937.723794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 915 00000100:00080000:3.0:1789760937.723804:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760937.723809:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419520:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789760937.723811:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760937.723821:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760937.723838:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507420032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760937.723850:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760937.723869:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760937/real 1789760937] req@ffff8d25d1b08e00 x1876699507419520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760937.723889:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507419520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760937.723898:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760937.723901:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760937.723919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760937.723951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760937.723975:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760937.723989:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760937/real 1789760937] req@ffff8d25d1b09880 x1876699507419648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760937.724002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507419648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760937.724011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760937.724014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760937.724018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760937.724025:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760937.724047:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760937.724059:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760937/real 1789760937] req@ffff8d25d1b0a680 x1876699507419776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760937.724071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507419776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760937.724085:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760937.724089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760937.724093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507419776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760937.724099:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507419904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760937.724121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760937.724133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760937/real 1789760937] req@ffff8d25d1b0b800 x1876699507419904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760937.724146:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507419904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760937.724155:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760937.724159:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760937.724162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507419904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760937.724169:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507420032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760937.724190:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760937.724203:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760937/real 1789760937] req@ffff8d25d1b08700 x1876699507420032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760937.724216:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507420032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760962 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760937.724223:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760937.724226:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760937.724231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507420032:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760937.787513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760938.811570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760939.835544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760940.859526:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760941.883515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760942.843430:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760942.843438:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760942.843442:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760942.843450:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 920 00000100:00080000:3.0:1789760942.843459:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760942.843465:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760942.843509:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507420160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760942.843529:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760942.843532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760942.843533:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760942.843537:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 920 00000100:00080000:3.0:1789760942.843540:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760942.843543:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760942.843551:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760942.843560:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507420288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760942.843564:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507420160:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789760942.843575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760942.843578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760942.843580:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760942.843583:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 920 00000100:00080000:3.0:1789760942.843586:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760942.843588:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760942.843591:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760942.843599:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507420416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760942.843606:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760942.843609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760942.843610:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760942.843612:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 920 00000100:00080000:3.0:1789760942.843615:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760942.843617:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760942.843620:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789760942.843623:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760942.843630:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507420544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760942.843638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760942.843641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760942.843643:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760942.843645:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 920 00000100:00080000:3.0:1789760942.843647:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760942.843650:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760942.843653:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760942.843661:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760942/real 1789760942] req@ffff8d25d1b08700 x1876699507420160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760942.843662:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507420672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760942.843692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760942.843695:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760942.843696:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789760942.843701:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507420160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760942.843701:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 920 00000100:00080000:3.0:1789760942.843704:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760942.843710:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760942.843715:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789760942.843719:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760942.843725:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760942.843726:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09f80 x1876699507420800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760942.843730:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507420160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760942.843758:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507420288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760942.843776:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760942.843787:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760942/real 1789760942] req@ffff8d25d1b0b800 x1876699507420288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760942.843798:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507420288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760942.843804:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760942.843807:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760942.843809:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507420288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760942.843814:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507420416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760942.843831:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760942.843839:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760942/real 1789760942] req@ffff8d25d1b0a680 x1876699507420416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760942.843847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507420416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760942.843853:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760942.843856:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760942.843858:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507420416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760942.843862:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507420544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760942.843874:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760942.843882:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760942/real 1789760942] req@ffff8d25d1b09880 x1876699507420544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760942.843891:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507420544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760942.843899:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760942.843903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760942.843908:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507420544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760942.843916:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507420672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760942.843940:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760942.843953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760942/real 1789760942] req@ffff8d25d1b08e00 x1876699507420672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760942.843970:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507420672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760942.843977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760942.843980:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760942.843983:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507420672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760942.843989:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507420800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760942.844009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760942.844021:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760942/real 1789760942] req@ffff8d25d1b09f80 x1876699507420800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760942.844034:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09f80 x1876699507420800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760967 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760942.844042:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760942.844044:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760942.844055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507420800:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760942.907515:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760943.931572:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760944.955597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760945.979498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760947.003578:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760947.963461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760947.963472:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760947.963476:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760947.963495:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 925 00000100:00080000:3.0:1789760947.963506:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760947.963513:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760947.963544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507420928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760947.963566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760947.963571:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760947.963573:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760947.963578:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 925 00000100:00080000:3.0:1789760947.963582:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760947.963586:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760947.963597:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760947.963611:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507421056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760947.963623:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507420928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789760947.963627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760947.963634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760947.963636:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760947.963642:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 925 00000100:00080000:3.0:1789760947.963646:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760947.963651:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760947.963657:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760947.963669:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507421184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760947.963676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760947.963679:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760947.963681:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760947.963683:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 925 00000100:00080000:3.0:1789760947.963687:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789760947.963691:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789760947.963692:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760947.963699:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789760947.963707:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507421312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760947.963714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760947.963717:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760947.963718:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760947.963720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 925 00000100:00080000:3.0:1789760947.963723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760947.963727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760947.963733:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760947.963737:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760947/real 1789760947] req@ffff8d25d1b0bb80 x1876699507420928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760947.963748:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507421440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760947.963762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760947.963763:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507420928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760947.963769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760947.963773:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760947.963784:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 925 00000100:00080000:0.0:1789760947.963785:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760947.963791:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760947.963794:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789760947.963799:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760947.963804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507420928:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789760947.963807:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760947.963823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507421568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760947.963835:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760947.963861:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760947.963874:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760947/real 1789760947] req@ffff8d25d1b09c00 x1876699507421056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760947.963887:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507421056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760947.963896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760947.963899:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760947.963903:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760947.963910:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760947.963928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760947.963937:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760947/real 1789760947] req@ffff8d25d1b09180 x1876699507421184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760947.963945:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507421184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760947.963951:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760947.963953:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760947.963956:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760947.963963:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760947.963982:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760947.963995:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760947/real 1789760947] req@ffff8d25d1b0aa00 x1876699507421312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760947.964009:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507421312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760947.964018:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760947.964021:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760947.964024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760947.964030:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507421440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760947.964051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760947.964062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760947/real 1789760947] req@ffff8d25d1b0b100 x1876699507421440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760947.964074:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507421440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760947.964081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760947.964084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760947.964087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507421440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760947.964110:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507421568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760947.964128:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760947.964138:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760947/real 1789760947] req@ffff8d25d1b0ad80 x1876699507421568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760947.964152:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507421568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760972 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760947.964160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760947.964164:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760947.964168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507421568:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760948.027507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760949.051581:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760950.075507:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760951.099529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760952.123550:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760953.083525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760953.083536:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760953.083540:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760953.083551:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 930 00000100:00080000:0.0:1789760953.083562:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760953.083570:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760953.083624:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507421696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760953.083660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760953.083663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760953.083664:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760953.083668:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 930 00000100:00080000:0.0:1789760953.083671:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760953.083674:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760953.083684:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760953.083696:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507421824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760953.083705:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760953.083708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789760953.083709:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507421696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789760953.083710:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760953.083718:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 930 00000100:00080000:0.0:1789760953.083722:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760953.083724:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760953.083727:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760953.083734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507421952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760953.083740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760953.083743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760953.083745:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760953.083747:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 930 00000100:00080000:0.0:1789760953.083749:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760953.083751:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760953.083754:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760953.083762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507422080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760953.083768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760953.083771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760953.083772:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760953.083775:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 930 00000100:00080000:0.0:1789760953.083781:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760953.083784:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760953.083788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789760953.083793:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760953.083797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507422208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760953.083804:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760953.083807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760953.083808:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760953.083811:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 930 00000100:00080000:0.0:1789760953.083813:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760953.083816:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760953.083818:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760953.083824:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507422336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760953.083853:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760953] req@ffff8d25d25b8380 x1876699507421696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760953.083869:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507421696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760953.083883:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760953.083887:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760953.083890:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507421696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789760953.083913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760953.083932:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760953.083942:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760953] req@ffff8d25d25bb800 x1876699507421824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760953.083950:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507421824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760953.083956:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760953.083958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760953.083960:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760953.083964:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760953.083975:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760953.083982:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760953] req@ffff8d25d25b9f80 x1876699507421952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760953.083989:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507421952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760953.083995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760953.083996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760953.083998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507421952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760953.084002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789760953.084014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760953.084020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760953] req@ffff8d25d25bad80 x1876699507422080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760953.084028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507422080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760953.084033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760953.084035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760953.084037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789760953.084040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507422208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760953.084061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760953.084069:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760953] req@ffff8d25d25b8700 x1876699507422208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760953.084076:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507422208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760953.084081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760953.084083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760953.084086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507422208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760953.084089:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507422336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760953.084099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760953.084105:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760953/real 1789760953] req@ffff8d25d25bb100 x1876699507422336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760953.084113:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507422336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760953.084118:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760953.084120:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760953.084121:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507422336:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760953.147514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760954.171529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760955.195481:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760956.219528:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760957.243550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760958.203437:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760958.203449:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760958.203460:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760958.203472:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 935 00000100:00080000:0.0:1789760958.203482:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760958.203490:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760958.203532:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507422464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760958.203561:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760958.203564:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760958.203565:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760958.203569:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 935 00000100:00080000:0.0:1789760958.203571:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760958.203575:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760958.203585:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760958.203599:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507422592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760958.203606:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507422464:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789760958.203610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760958.203614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760958.203616:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760958.203620:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 935 00000100:00080000:0.0:1789760958.203622:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760958.203625:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760958.203629:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760958.203638:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507422720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760958.203645:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760958.203648:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760958.203649:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760958.203652:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 935 00000100:00080000:0.0:1789760958.203655:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760958.203659:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760958.203663:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760958.203671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507422848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760958.203679:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760958.203682:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760958.203683:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760958.203685:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 935 00000100:00080000:0.0:1789760958.203688:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789760958.203693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789760958.203694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760958.203700:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760958.203708:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507422976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760958.203714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760958.203718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760958.203719:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760958.203722:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 935 00000100:00080000:0.0:1789760958.203725:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760958.203727:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760958.203730:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760958.203737:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507423104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789760958.203749:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffff8d25d25b9c00 x1876699507422464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760958.203767:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507422464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760958.203777:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760958.203781:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760958.203784:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507422464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789760958.203806:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760958.203824:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760958.203833:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffff8d25d25b8e00 x1876699507422592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760958.203847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507422592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760958.203853:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760958.203855:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760958.203857:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760958.203860:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760958.203880:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760958.203888:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffff8d25d25bb480 x1876699507422720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760958.203895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507422720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760958.203900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760958.203903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760958.203905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760958.203908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789760958.203920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760958.203926:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffff8d25d25bb100 x1876699507422848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760958.203933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507422848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760958.203942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760958.203944:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760958.203946:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507422848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789760958.203949:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507422976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760958.203960:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760958.203966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffff8d25d25b8700 x1876699507422976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760958.203973:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507422976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760958.203978:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760958.203979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760958.203981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507422976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760958.203984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507423104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760958.203994:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760958.204000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760958/real 1789760958] req@ffff8d25d25bad80 x1876699507423104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760958.204007:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507423104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760958.204012:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760958.204013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760958.204016:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507423104:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760958.267592:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760959.291509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760960.315632:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760961.339522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760962.363588:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760963.323516:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760963.323525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760963.323529:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760963.323540:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 941 00000100:00080000:3.0:1789760963.323550:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760963.323557:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760963.323591:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507423232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760963.323617:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760963.323620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760963.323622:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760963.323625:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 941 00000100:00080000:3.0:1789760963.323629:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760963.323632:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760963.323641:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760963.323651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507423360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760963.323660:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507423232:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789760963.323663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760963.323669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760963.323671:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760963.323675:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 941 00000100:00080000:3.0:1789760963.323677:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760963.323680:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760963.323684:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760963.323694:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507423488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760963.323701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760963.323704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760963.323705:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760963.323708:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 941 00000100:00080000:3.0:1789760963.323711:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760963.323713:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760963.323717:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789760963.323725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507423616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760963.323730:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789760963.323741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760963.323744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760963.323746:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760963.323751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 941 00000100:00080000:3.0:1789760963.323754:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760963.323757:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760963.323761:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760963.323769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507423744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760963.323779:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffff8d25d1b08a80 x1876699507423232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760963.323780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760963.323788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760963.323791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760963.323800:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 941 00000100:00080000:3.0:1789760963.323805:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760963.323808:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760963.323812:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760963.323814:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507423232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760963.323823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507423872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760963.323825:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760963.323835:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760963.323844:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507423232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760963.323868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507423360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760963.323885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760963.323894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffff8d25d1b09500 x1876699507423360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760963.323903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507423360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760963.323910:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760963.323912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760963.323915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507423360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760963.323919:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507423488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760963.323931:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760963.323938:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffff8d25d1b08000 x1876699507423488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760963.323946:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507423488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760963.323952:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760963.323954:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760963.323956:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507423488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760963.323960:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507423616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760963.323971:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760963.323977:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffff8d25d1b0b480 x1876699507423616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760963.323985:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507423616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760963.323991:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760963.323993:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760963.323995:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507423616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760963.323999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507423744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760963.324010:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760963.324016:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffff8d25d1b0a300 x1876699507423744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760963.324030:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507423744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760963.324036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760963.324037:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760963.324039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507423744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760963.324043:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507423872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760963.324054:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760963.324060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760963/real 1789760963] req@ffff8d25d1b08380 x1876699507423872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760963.324067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507423872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760963.324073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760963.324075:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760963.324077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507423872:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760963.387545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760964.411547:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760965.435582:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760966.459588:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760967.483510:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760968.443507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760968.443525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760968.443533:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760968.443549:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 946 00000100:00080000:3.0:1789760968.443564:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760968.443577:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760968.443654:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507424000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760968.443701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760968.443707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760968.443710:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760968.443718:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 946 00000100:00080000:3.0:1789760968.443724:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760968.443731:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760968.443749:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760968.443769:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507424000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789760968.443772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507424128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760968.443794:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760968.443800:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760968.443802:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760968.443809:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 946 00000100:00080000:3.0:1789760968.443815:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760968.443822:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760968.443830:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760968.443849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507424256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760968.443862:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789760968.443879:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760968.443899:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760968.443903:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760968.443912:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 946 00000100:00080000:3.0:1789760968.443919:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760968.443926:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760968.443931:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760968/real 1789760968] req@ffff8d25d1b08380 x1876699507424000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789760968.443937:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789760968.443967:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507424384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760968.443972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507424000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760968.444006:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760968.444008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760968.444017:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760968.444018:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789760968.444026:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789760968.444028:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507424000:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789760968.444042:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 946 00000100:00080000:3.0:1789760968.444050:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760968.444056:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760968.444065:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760968.444069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760968.444091:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507424512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760968.444113:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789760968.444116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789760968.444124:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760968.444128:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760968.444138:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 946 00000100:00100000:0.0:1789760968.444140:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760968/real 1789760968] req@ffff8d25d1b0a300 x1876699507424128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760968.444154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507424128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760968.444162:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760968.444167:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760968.444175:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760968.444185:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424128:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789760968.444185:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760968.444200:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424256:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789760968.444202:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789760968.444229:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760968.444229:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507424640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760968.444245:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760968/real 1789760968] req@ffff8d25d1b0b480 x1876699507424256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760968.444272:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507424256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760968.444281:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760968.444285:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760968.444308:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760968.444332:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760968.444366:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760968.444426:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760968/real 1789760968] req@ffff8d25d1b08000 x1876699507424384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760968.444441:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507424384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760968.444449:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760968.444452:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760968.444455:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760968.444478:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507424512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760968.444501:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760968.444517:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760968/real 1789760968] req@ffff8d25d1b09500 x1876699507424512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760968.444535:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507424512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760968.444548:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760968.444553:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760968.444558:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507424512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760968.444566:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507424640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760968.444587:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760968.444599:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760968/real 1789760968] req@ffff8d25d1b08a80 x1876699507424640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760968.444610:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507424640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760968.444618:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760968.444621:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760968.444624:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507424640:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760968.507548:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760969.531554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760970.555579:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760971.579570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760972.603566:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760973.563610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760973.563621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760973.563626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760973.563638:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 951 00000100:00080000:2.0:1789760973.563653:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760973.563662:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760973.563715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507424768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760973.563746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760973.563750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760973.563752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760973.563756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 951 00000100:00080000:2.0:1789760973.563759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760973.563762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760973.563772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760973.563784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507424896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760973.563793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760973.563795:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760973.563796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760973.563799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 951 00000100:00080000:2.0:1789760973.563802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760973.563808:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507424768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789760973.563808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760973.563812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760973.563824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507425024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760973.563830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760973.563833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760973.563834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760973.563837:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 951 00000100:00080000:2.0:1789760973.563839:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760973.563841:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760973.563845:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789760973.563853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507425152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760973.563860:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760973.563862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760973.563864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760973.563866:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 951 00000100:00080000:2.0:1789760973.563869:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760973.563871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760973.563874:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760973.563880:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507425280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760973.563886:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760973.563889:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760973.563897:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789760973.563903:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789760973.563904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 951 00000100:00080000:2.0:1789760973.563909:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760973.563912:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760973.563915:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760973.563924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507425408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760973.563962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760973/real 1789760973] req@ffff8d25c8023100 x1876699507424768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760973.563984:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507424768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760973.563998:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760973.564003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760973.564007:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507424768:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760973.564048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760973.564081:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760973.564096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760973/real 1789760973] req@ffff8d25c8023b80 x1876699507424896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760973.564111:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507424896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760973.564127:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760973.564131:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760973.564135:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507424896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760973.564143:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760973.564164:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760973.564177:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760973/real 1789760973] req@ffff8d25c8020700 x1876699507425024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760973.564190:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507425024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760973.564199:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760973.564203:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760973.564206:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760973.564214:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760973.564234:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760973.564247:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760973/real 1789760973] req@ffff8d25c8022d80 x1876699507425152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760973.564277:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507425152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760973.564287:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760973.564291:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760973.564296:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760973.564303:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507425280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760973.564324:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760973.564337:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760973/real 1789760973] req@ffff8d25c8021180 x1876699507425280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760973.564350:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507425280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760973.564358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760973.564362:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760973.564366:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507425280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760973.564373:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507425408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760973.564440:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760973.564454:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760973/real 1789760973] req@ffff8d25c8020000 x1876699507425408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760973.564468:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507425408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789760998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760973.564477:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760973.564480:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760973.564484:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507425408:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760973.627580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760974.651511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760975.675562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760976.699532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760977.723644:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789760978.683493:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760978.683501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789760978.683506:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760978.683518:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 956 00000100:00080000:3.0:1789760978.683528:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760978.683536:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760978.683576:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507425536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760978.683600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760978.683603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760978.683605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760978.683609:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 956 00000100:00080000:3.0:1789760978.683612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760978.683615:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760978.683637:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760978.683651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507425664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760978.683652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507425536:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789760978.683667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760978.683670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760978.683671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760978.683675:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 956 00000100:00080000:3.0:1789760978.683678:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760978.683682:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760978.683687:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760978.683707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507425792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760978.683715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760978.683718:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789760978.683720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760978.683723:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 956 00000100:00080000:3.0:1789760978.683725:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789760978.683728:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760978.683732:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789760978.683743:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760978.683744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507425920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760978.683753:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760978.683756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789760978.683757:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760978.683760:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 956 00000100:00080000:3.0:1789760978.683762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760978.683795:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760978/real 1789760978] req@ffff8d25d1b0ad80 x1876699507425536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760978.683813:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507425536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760978.683834:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760978.683838:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789760978.683841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760978.683846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760978.683848:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507425536:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789760978.683859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507426048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760978.683866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789760978.683869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789760978.683871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789760978.683875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 956 00000100:00080000:3.0:1789760978.683878:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760978.683879:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425664:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789760978.683884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789760978.683889:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760978.683899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507426176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760978.683917:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760978.683930:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760978/real 1789760978] req@ffff8d25d1b0b100 x1876699507425664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760978.683943:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507425664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760978.683953:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760978.683956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760978.683960:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760978.683967:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760978.683989:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760978.684001:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760978/real 1789760978] req@ffff8d25d1b0aa00 x1876699507425792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760978.684019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507425792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760978.684028:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760978.684032:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760978.684036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760978.684043:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760978.684076:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760978.684085:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760978/real 1789760978] req@ffff8d25d1b09180 x1876699507425920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760978.684094:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507425920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760978.684100:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760978.684102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760978.684105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507425920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760978.684110:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760978.684123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760978.684131:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760978/real 1789760978] req@ffff8d25d1b09c00 x1876699507426048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760978.684139:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507426048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760978.684150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760978.684152:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760978.684156:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760978.684175:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760978.684188:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760978.684194:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760978/real 1789760978] req@ffff8d25d1b0bb80 x1876699507426176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760978.684205:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507426176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761003 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760978.684227:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760978.684231:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760978.684236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426176:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760978.747505:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760979.771527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760980.795563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760981.819520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760982.843498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760983.803612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760983.803642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760983.803649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760983.803663:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 961 00000100:00080000:2.0:1789760983.803677:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760983.803686:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760983.803750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507426304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760983.803785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760983.803788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760983.803790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760983.803794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 961 00000100:00080000:2.0:1789760983.803797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760983.803801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760983.803817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760983.803830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507426432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760983.803839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760983.803843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760983.803845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760983.803849:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 961 00000100:00080000:2.0:1789760983.803852:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760983.803856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760983.803859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760983.803870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507426560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760983.803889:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507426304:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789760983.803891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760983.803897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760983.803903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760983.803913:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 961 00000100:00080000:2.0:1789760983.803917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760983.803920:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760983.803924:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789760983.803941:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507426688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760983.803955:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760983.803958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760983.803960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760983.803963:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 961 00000100:00080000:2.0:1789760983.803966:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760983.803986:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760983.803991:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760983.804005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507426816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789760983.804023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789760983.804026:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760983.804053:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760983.804056:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760983.804063:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 961 00000100:00080000:2.0:1789760983.804070:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760983.804074:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760983.804081:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760983.804096:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507426944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760983.804124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760983/real 1789760983] req@ffff8d25c8021500 x1876699507426304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760983.804154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507426304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760983.804190:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760983.804209:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760983.804215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507426304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760983.804228:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507426432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760983.804264:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760983.804284:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760983/real 1789760983] req@ffff8d25c8023480 x1876699507426432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760983.804303:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507426432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760983.804315:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760983.804321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760983.804326:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507426432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760983.804360:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507426560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760983.804431:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760983.804452:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760983/real 1789760983] req@ffff8d25c8021880 x1876699507426560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760983.804469:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507426560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760983.804481:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760983.804486:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760983.804491:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507426560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760983.804501:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507426688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760983.804530:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760983.804546:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760983/real 1789760983] req@ffff8d25c8021c00 x1876699507426688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760983.804562:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507426688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760983.804573:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760983.804577:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760983.804581:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507426688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760983.804590:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760983.804612:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760983.804625:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760983/real 1789760983] req@ffff8d25c8020e00 x1876699507426816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760983.804642:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507426816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760983.804653:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760983.804657:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760983.804662:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760983.804670:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760983.804694:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760983.804707:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760983/real 1789760983] req@ffff8d25c8020000 x1876699507426944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760983.804731:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507426944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761008 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760983.804741:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760983.804745:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760983.804750:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507426944:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760983.867619:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760984.891562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760985.915563:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760986.939552:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760987.963563:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789760988.923624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760988.923637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789760988.923643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760988.923657:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 966 00000100:00080000:0.0:1789760988.923673:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760988.923685:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789760988.923738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507427072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760988.923788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760988.923794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760988.923798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760988.923807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 966 00000100:00080000:0.0:1789760988.923816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760988.923824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760988.923840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760988.923852:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507427072:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789760988.923871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507427200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760988.923892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760988.923897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760988.923899:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760988.923904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 966 00000100:00080000:0.0:1789760988.923909:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760988.923914:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760988.923920:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760988.923935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507427328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760988.923946:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760988.923950:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789760988.923951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760988.923955:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 966 00000100:00080000:0.0:1789760988.923959:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789760988.923963:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789760988.923970:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789760988.923978:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789760988.923993:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507427456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789760988.924005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789760988.924008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789760988.924011:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760988.924018:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 966 00000100:00080000:0.0:1789760988.924024:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760988.924037:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760988.924038:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760988/real 1789760988] req@ffff8d25d25b9f80 x1876699507427072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789760988.924051:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789760988.924085:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507427072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760988.924088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507427584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789760988.924115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760988.924122:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789760988.924130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789760988.924131:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507427072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789760988.924175:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789760988.924179:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789760988.924186:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 966 00000100:00080000:0.0:1789760988.924192:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789760988.924197:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789760988.924205:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427200:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789760988.924209:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760988.924261:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507427712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789760988.924277:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760988.924307:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760988/real 1789760988] req@ffff8d25d25bb800 x1876699507427200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760988.924320:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507427200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760988.924327:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760988.924331:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760988.924333:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760988.924339:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760988.924357:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760988.924365:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760988/real 1789760988] req@ffff8d25d25b8380 x1876699507427328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760988.924452:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507427328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760988.924459:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760988.924462:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760988.924465:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760988.924483:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789760988.924505:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760988.924530:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760988/real 1789760988] req@ffff8d25d25b9180 x1876699507427456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760988.924543:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507427456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760988.924549:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760988.924551:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760988.924553:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789760988.924557:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507427584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760988.924570:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760988.924577:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760988/real 1789760988] req@ffff8d25d25ba680 x1876699507427584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760988.924612:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507427584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760988.924618:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760988.924621:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760988.924623:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507427584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789760988.924641:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507427712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789760988.924672:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789760988.924680:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760988/real 1789760988] req@ffff8d25d25bbb80 x1876699507427712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789760988.924687:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507427712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761013 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789760988.924697:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789760988.924699:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789760988.924701:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507427712:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789760988.987558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760990.011545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760991.035577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760992.059577:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760993.083664:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760994.043603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.043613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760994.043618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.043643:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 971 00000100:00080000:2.0:1789760994.043651:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.043658:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760994.043710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507427840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760994.043740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.043744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760994.043747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.043752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 971 00000100:00080000:2.0:1789760994.043756:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.043761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760994.043770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760994.043781:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507427968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760994.043791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789760994.043798:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507427840:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789760994.043800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760994.043807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.043816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 971 00000100:00080000:2.0:1789760994.043822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.043828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760994.043834:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760994.043850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507428096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760994.043864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.043869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760994.043872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.043877:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 971 00000100:00080000:2.0:1789760994.043882:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789760994.043885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789760994.043892:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760994.043901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789760994.043930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507428224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760994.043943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffff8d25c8021180 x1876699507427840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760994.043945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.043979:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760994.043996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.044009:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 971 00000100:00100000:0.0:1789760994.044015:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507427840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789760994.044021:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.044040:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789760994.044041:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.044054:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789760994.044060:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760994.044062:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507427840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789760994.044088:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507428352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760994.044140:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760994.044143:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760994.044148:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760994.044150:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760994.044155:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 971 00000100:00080000:2.0:1789760994.044162:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760994.044166:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789760994.044172:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789760994.044175:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789760994.044188:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffff8d25c8022d80 x1876699507427968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789760994.044203:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507428480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760994.044207:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507427968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.044235:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.044240:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760994.044243:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507427968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760994.044250:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760994.044269:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760994.044278:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffff8d25c8020700 x1876699507428096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760994.044287:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507428096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.044293:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.044295:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760994.044297:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760994.044302:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789760994.044314:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760994.044322:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffff8d25c8023b80 x1876699507428224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760994.044331:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507428224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.044337:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.044339:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760994.044341:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760994.044355:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507428352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760994.044367:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760994.044375:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffff8d25c8023100 x1876699507428352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760994.044384:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507428352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.044390:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.044413:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760994.044416:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507428352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760994.044421:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507428480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789760994.044436:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789760994.044445:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760994/real 1789760994] req@ffff8d25c8022a00 x1876699507428480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789760994.044460:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507428480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789760994.044466:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789760994.044468:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789760994.044471:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507428480:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760994.107546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760995.131561:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760996.155549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789760997.179669:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789760998.203560:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789760999.163478:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760999.163487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789760999.163491:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760999.163500:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 976 00000100:00080000:2.0:1789760999.163509:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760999.163516:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789760999.163547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507428608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760999.163571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760999.163575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760999.163576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760999.163580:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 976 00000100:00080000:2.0:1789760999.163583:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760999.163587:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760999.163595:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760999.163606:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507428736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760999.163614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760999.163617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760999.163619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760999.163623:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 976 00000100:00080000:2.0:1789760999.163630:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760999.163632:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507428608:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789760999.163634:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760999.163647:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760999.163662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507428864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760999.163674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760999.163678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789760999.163681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760999.163685:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 976 00000100:00080000:2.0:1789760999.163690:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760999.163696:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760999.163702:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789760999.163722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507428992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760999.163735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760999.163738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789760999.163740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760999.163742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 976 00000100:00080000:2.0:1789760999.163747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789760999.163750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760999.163755:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760999.163765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507429120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789760999.163775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789760999.163778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789760999.163780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789760999.163786:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 976 00000100:00080000:2.0:1789760999.163789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789760999.163793:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428736:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789760999.163796:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789760999.163803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789760999.163817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507429248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789760999.163850:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760999.163874:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789760999.163889:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507429120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789760999.163911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507429248:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789760999.227563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761000.251511:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761001.275580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761002.235765:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761002.235825:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761002.235851:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761002.235919:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761002.235935:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761002.235951:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761002.235963:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761002.235977:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761002.236143:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789761002] req@ffff8d25c8022a00 x1876699507429248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761002.236186:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507429248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761002.236205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761002.236214:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761002.236221:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507429248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761002.236234:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789761002] req@ffff8d25c8022680 x1876699507428736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761002.236253:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507428736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761002.236265:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761002.236271:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761002.236276:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761002.236288:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789761002] req@ffff8d25c8023800 x1876699507428864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761002.236305:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507428864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761002.236317:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761002.236323:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761002.236329:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761002.236339:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789761002] req@ffff8d25c8020380 x1876699507428992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761002.236358:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507428992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761002.236371:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761002.236442:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761002.236450:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507428992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761002.236473:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789761002] req@ffff8d25c8020a80 x1876699507429120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761002.236494:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507429120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761002.236507:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761002.236512:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761002.236517:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507429120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761002.236527:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789760999/real 1789761002] req@ffff8d25c8022300 x1876699507428608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761002.236572:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507428608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761002.236584:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761002.236595:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761002.236600:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507428608:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1789761002.299530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761003.323521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761004.283546:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.283557:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761004.283563:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.283576:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 981 00000100:00080000:2.0:1789761004.283593:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.283604:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761004.283646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507429376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761004.283681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.283685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761004.283687:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.283692:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 981 00000100:00080000:2.0:1789761004.283696:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.283700:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761004.283721:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761004.283734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507429504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761004.283746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.283750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761004.283753:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.283758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 981 00000100:00080000:2.0:1789761004.283761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.283765:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761004.283770:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761004.283781:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507429376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761004.283785:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507429632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761004.283811:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.283815:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761004.283817:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.283821:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 981 00000100:00080000:2.0:1789761004.283824:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.283827:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761004.283831:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761004.283841:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507429760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761004.283850:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.283853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761004.283854:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.283859:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 981 00000100:00080000:2.0:1789761004.283862:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.283867:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761004.283871:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761004.283881:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507429888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761004.283898:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761004.283902:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761004.283904:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761004.283909:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 981 00000100:00080000:2.0:1789761004.283913:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761004.283917:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761004.283922:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761004.283951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761004.283954:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507430016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761004.284066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffff8d25c8023100 x1876699507429376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.284103:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507429376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.284123:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.284131:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761004.284138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507429376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761004.284153:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507429504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761004.284198:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761004.284222:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffff8d25c8023b80 x1876699507429504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.284244:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507429504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.284257:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.284263:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761004.284269:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507429504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761004.284330:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507429632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761004.284370:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761004.284441:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffff8d25c8020700 x1876699507429632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.284463:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507429632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.284476:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.284482:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761004.284488:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507429632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761004.284501:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507429760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761004.284539:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761004.284560:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffff8d25c8022d80 x1876699507429760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.284582:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507429760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.284596:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.284601:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761004.284607:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507429760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761004.284617:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507429888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761004.284658:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761004.284679:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffff8d25c8021180 x1876699507429888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.284699:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507429888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.284712:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.284718:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761004.284723:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507429888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761004.284734:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507430016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761004.284769:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761004.284790:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761004/real 1789761004] req@ffff8d25c8020000 x1876699507430016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761004.284810:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507430016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761004.284822:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761004.284827:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761004.284832:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507430016:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761004.347573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761005.371577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761006.395529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761007.419559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761008.443519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761009.403472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761009.403479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761009.403483:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761009.403498:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 987 00000100:00080000:0.0:1789761009.403506:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761009.403512:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761009.403543:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507430144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761009.403566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761009.403569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761009.403570:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761009.403573:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 987 00000100:00080000:0.0:1789761009.403575:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761009.403578:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761009.403584:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761009.403595:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507430272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761009.403602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761009.403605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761009.403606:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761009.403610:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 987 00000100:00080000:0.0:1789761009.403613:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761009.403614:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507430144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761009.403617:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761009.403623:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761009.403631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507430400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761009.403637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761009.403640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761009.403641:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761009.403647:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 987 00000100:00080000:0.0:1789761009.403650:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761009.403654:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761009.403659:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761009.403674:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761009.403677:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507430528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761009.403690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761009.403694:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761009.403696:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761009.403702:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 987 00000100:00080000:0.0:1789761009.403706:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761009.403711:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761009/real 1789761009] req@ffff8d25d25b8a80 x1876699507430144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761009.403715:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761009.403726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761009.403733:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507430144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761009.403740:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507430656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761009.403744:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761009.403752:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761009.403753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761009.403758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761009.403760:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507430144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761009.403761:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761009.403768:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 987 00000100:00080000:0.0:1789761009.403772:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761009.403774:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761009.403777:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761009.403783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507430784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761009.403791:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507430272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761009.403820:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761009.403832:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761009/real 1789761009] req@ffff8d25d25ba300 x1876699507430272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761009.403841:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507430272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761009.403847:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761009.403850:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761009.403853:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507430272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761009.403857:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507430400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761009.403870:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761009.403876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761009/real 1789761009] req@ffff8d25d25b8000 x1876699507430400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761009.403884:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507430400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761009.403890:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761009.403892:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761009.403894:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507430400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761009.403897:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507430528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761009.403908:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761009.403914:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761009/real 1789761009] req@ffff8d25d25bbb80 x1876699507430528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761009.403922:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507430528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761009.403928:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761009.403929:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761009.403931:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507430528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761009.403935:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507430656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761009.403945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761009.403951:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761009/real 1789761009] req@ffff8d25d25ba680 x1876699507430656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761009.403959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507430656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761009.403963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761009.403965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761009.403967:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507430656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761009.403979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507430784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761009.404011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761009.404018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761009/real 1789761009] req@ffff8d25d25b9180 x1876699507430784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761009.404025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507430784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761009.404035:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761009.404038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761009.404040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507430784:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761009.467525:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761010.491488:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761011.515557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761012.539575:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761013.563551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761014.523500:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761014.523512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761014.523516:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761014.523533:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 992 00000100:00080000:0.0:1789761014.523545:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761014.523553:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761014.523595:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507430912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761014.523621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761014.523624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761014.523626:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761014.523629:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 992 00000100:00080000:0.0:1789761014.523632:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761014.523634:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761014.523644:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761014.523656:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507431040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761014.523664:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507430912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761014.523667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761014.523674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761014.523675:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761014.523679:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 992 00000100:00080000:0.0:1789761014.523681:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761014.523684:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761014.523687:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761014.523696:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507431168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761014.523708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761014.523711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761014.523712:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761014.523715:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 992 00000100:00080000:0.0:1789761014.523719:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761014.523721:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761014.523726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761014.523734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507431296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761014.523742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761014.523748:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761014.523749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761014.523752:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761014.523757:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 992 00000100:00080000:0.0:1789761014.523759:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761014.523763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761014.523766:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761014.523773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507431424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761014.523779:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761014.523782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761014.523784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761014.523787:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 992 00000100:00080000:0.0:1789761014.523791:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761014.523794:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761014.523799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761014.523813:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507431552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761014.523819:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761014/real 1789761014] req@ffff8d25d25b8380 x1876699507430912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761014.523858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507430912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761014.523869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761014.523873:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761014.523876:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507430912:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761014.523901:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761014.523924:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761014.523936:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761014/real 1789761014] req@ffff8d25d25bb800 x1876699507431040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761014.523961:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507431040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761014.523972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761014.523975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761014.523977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761014.523983:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761014.524009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761014.524023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761014/real 1789761014] req@ffff8d25d25b9f80 x1876699507431168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761014.524039:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507431168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761014.524047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761014.524050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761014.524053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761014.524058:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761014.524078:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761014.524087:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761014/real 1789761014] req@ffff8d25d25bad80 x1876699507431296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761014.524096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507431296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761014.524102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761014.524105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761014.524108:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761014.524113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507431424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761014.524196:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761014.524207:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761014/real 1789761014] req@ffff8d25d25b8700 x1876699507431424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761014.524218:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507431424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761014.524224:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761014.524227:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761014.524230:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507431424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761014.524243:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507431552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761014.524257:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761014.524265:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761014/real 1789761014] req@ffff8d25d25bb100 x1876699507431552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761014.524273:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507431552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761014.524278:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761014.524280:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761014.524281:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507431552:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761014.587636:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761015.611589:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761016.635503:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761017.659532:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761018.683566:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761019.643500:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761019.643510:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761019.643516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761019.643537:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 997 00000100:00080000:2.0:1789761019.643549:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761019.643557:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761019.643596:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507431680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761019.643620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761019.643623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761019.643624:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761019.643628:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 997 00000100:00080000:2.0:1789761019.643630:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761019.643633:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761019.643644:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761019.643655:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507431808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761019.643665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761019.643671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761019.643673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761019.643673:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507431680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761019.643687:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 997 00000100:00080000:2.0:1789761019.643691:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761019.643695:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761019.643698:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761019.643706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507431936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761019.643713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761019.643716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761019.643717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761019.643720:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 997 00000100:00080000:2.0:1789761019.643723:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761019.643726:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761019.643729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761019.643736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507432064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761019.643743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761019.643746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761019.643747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761019.643749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 997 00000100:00080000:2.0:1789761019.643752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761019.643755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761019.643766:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761019.643768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761019.643780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507432192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761019.643791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761019.643794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761019.643795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761019.643799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 997 00000100:00080000:2.0:1789761019.643802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761019.643810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761019.643810:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761019/real 1789761019] req@ffff8d25c8020e00 x1876699507431680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761019.643819:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761019.643841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507432320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761019.643856:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507431680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761019.643866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761019.643870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761019.643874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507431680:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761019.643897:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761019.643936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761019.643951:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761019/real 1789761019] req@ffff8d25c8021c00 x1876699507431808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761019.643965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507431808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761019.643973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761019.643975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761019.643978:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761019.643983:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761019.643999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761019.644009:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761019/real 1789761019] req@ffff8d25c8021880 x1876699507431936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761019.644017:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507431936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761019.644023:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761019.644027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761019.644031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507431936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761019.644037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761019.644059:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761019.644071:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761019/real 1789761019] req@ffff8d25c8023480 x1876699507432064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761019.644084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507432064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761019.644090:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761019.644093:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761019.644095:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761019.644100:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507432192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761019.644115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761019.644123:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761019/real 1789761019] req@ffff8d25c8021500 x1876699507432192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761019.644130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507432192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761019.644136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761019.644138:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761019.644139:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507432192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761019.644143:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507432320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761019.644154:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761019.644160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761019/real 1789761019] req@ffff8d25c8020000 x1876699507432320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761019.644166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507432320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761019.644171:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761019.644174:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761019.644176:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507432320:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761019.707510:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761020.731520:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761021.755506:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761022.779610:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761023.803502:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761024.763482:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761024.763497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761024.763503:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761024.763515:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1002 00000100:00080000:2.0:1789761024.763525:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761024.763533:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761024.763585:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507432448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761024.763612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761024.763615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761024.763617:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761024.763622:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1002 00000100:00080000:2.0:1789761024.763627:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761024.763632:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761024.763646:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761024.763646:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507432448:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761024.763665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507432576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761024.763676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761024.763680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761024.763682:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761024.763686:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1002 00000100:00080000:2.0:1789761024.763689:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761024.763692:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761024.763697:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761024.763706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507432704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761024.763714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761024.763718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761024.763720:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761024.763724:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761024.763729:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1002 00000100:00080000:2.0:1789761024.763733:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761024.763738:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761024.763744:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761024.763756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507432832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761024.763766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761024.763769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761024.763771:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761024.763774:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1002 00000100:00080000:2.0:1789761024.763778:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761024.763781:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761024/real 1789761024] req@ffff8d25c8021180 x1876699507432448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761024.763786:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761024.763799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761024.763810:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507432448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761024.763816:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507432960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761024.763853:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761024.763853:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761024.763860:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761024.763861:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761024.763880:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507432448:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761024.763886:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761024.763894:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1002 00000100:00080000:2.0:1789761024.763899:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761024.763904:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761024.763907:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432576:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789761024.763913:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761024.763932:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507433088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761024.763935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761024.763949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761024/real 1789761024] req@ffff8d25c8022d80 x1876699507432576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761024.763961:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507432576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761024.763968:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761024.763971:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761024.763974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761024.763978:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761024.763991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761024.763997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761024/real 1789761024] req@ffff8d25c8020700 x1876699507432704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761024.764005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507432704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761024.764010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761024.764012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761024.764014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761024.764018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761024.764030:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761024.764036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761024/real 1789761024] req@ffff8d25c8023b80 x1876699507432832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761024.764043:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507432832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761024.764049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761024.764051:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761024.764053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507432832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761024.764056:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507432960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761024.764067:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761024.764081:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761024/real 1789761024] req@ffff8d25c8023100 x1876699507432960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761024.764089:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507432960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761024.764095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761024.764097:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761024.764099:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507432960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761024.764109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507433088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761024.764121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761024.764128:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761024/real 1789761024] req@ffff8d25c8022300 x1876699507433088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761024.764136:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507433088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761049 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761024.764141:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761024.764142:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761024.764144:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507433088:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761024.827551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761025.851546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761026.875549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761027.899499:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761028.923536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761029.883478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761029.883489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761029.883493:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761029.883504:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1007 00000100:00080000:0.0:1789761029.883514:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761029.883522:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761029.883558:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507433216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761029.883583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761029.883588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761029.883590:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761029.883594:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1007 00000100:00080000:0.0:1789761029.883598:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761029.883602:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761029.883612:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761029.883625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507433216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761029.883629:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507433344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761029.883643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761029.883647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761029.883650:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761029.883654:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1007 00000100:00080000:0.0:1789761029.883658:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761029.883662:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761029.883668:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761029.883680:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507433472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761029.883692:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761029.883694:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761029.883697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761029.883700:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761029.883706:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1007 00000100:00080000:0.0:1789761029.883710:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761029.883714:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761029.883720:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761029.883733:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507433600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761029.883745:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761029/real 1789761029] req@ffff8d25d25bb480 x1876699507433216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761029.883746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761029.883755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761029.883759:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761029.883767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1007 00000100:00080000:0.0:1789761029.883771:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761029.883775:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507433216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761029.883778:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761029.883785:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761029.883790:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761029.883795:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761029.883800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507433728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761029.883801:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507433216:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761029.883818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761029.883821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761029.883823:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761029.883840:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1007 00000100:00080000:0.0:1789761029.883844:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761029.883846:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507433344:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761029.883851:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761029.883857:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761029.883871:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507433856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761029.883877:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761029.883899:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761029/real 1789761029] req@ffff8d25d25b8e00 x1876699507433344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761029.883916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507433344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761029.883923:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761029.883926:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761029.883928:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507433344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761029.883932:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507433472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761029.883945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761029.883952:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761029/real 1789761029] req@ffff8d25d25b9c00 x1876699507433472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761029.883960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507433472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761029.883966:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761029.883968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761029.883970:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507433472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761029.883973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507433600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761029.883983:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761029.883989:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761029/real 1789761029] req@ffff8d25d25bb100 x1876699507433600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761029.883997:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507433600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761029.884007:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761029.884009:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761029.884011:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507433600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761029.884015:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507433728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761029.884024:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761029.884030:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761029/real 1789761029] req@ffff8d25d25b8700 x1876699507433728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761029.884038:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507433728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761029.884043:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761029.884045:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761029.884047:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507433728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761029.884059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507433856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761029.884070:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761029.884077:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761029/real 1789761029] req@ffff8d25d25bad80 x1876699507433856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761029.884084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507433856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761054 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761029.884089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761029.884091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761029.884094:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507433856:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761029.947518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761030.971510:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761031.995544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761033.019585:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761034.043552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761035.003520:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761035.003530:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761035.003534:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.003559:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1012 00000100:00080000:2.0:1789761035.003569:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.003587:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761035.003625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507433984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761035.003694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761035.003697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761035.003699:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.003715:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1012 00000100:00080000:2.0:1789761035.003717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.003720:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761035.003729:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761035.003742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507434112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761035.003752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761035.003752:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507433984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761035.003758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761035.003763:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.003767:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1012 00000100:00080000:2.0:1789761035.003768:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.003771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761035.003784:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761035.003809:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507434240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761035.003818:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761035.003821:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761035.003823:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.003844:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1012 00000100:00080000:2.0:1789761035.003850:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.003857:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761035.003860:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761035.003870:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761035.003888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507434368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761035.003902:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761035.003903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffff8d25c8020a80 x1876699507433984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761035.003914:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761035.003919:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761035.003928:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1012 00000100:00100000:3.0:1789761035.003932:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507433984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761035.003935:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.003969:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761035.003973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789761035.003976:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761035.003980:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761035.003985:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507433984:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761035.003992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507434496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761035.004001:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761035.004005:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761035.004007:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761035.004010:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434112:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761035.004014:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1012 00000100:00080000:2.0:1789761035.004019:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761035.004032:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761035.004035:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761035.004045:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507434624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761035.004050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761035.004067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffff8d25c8020380 x1876699507434112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761035.004080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507434112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761035.004087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761035.004090:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761035.004092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761035.004098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761035.004110:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761035.004118:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffff8d25c8023800 x1876699507434240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761035.004126:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507434240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761035.004132:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761035.004134:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761035.004136:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761035.004140:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761035.004151:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761035.004158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffff8d25c8022680 x1876699507434368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761035.004166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507434368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761035.004172:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761035.004174:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761035.004176:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761035.004180:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507434496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761035.004190:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761035.004197:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffff8d25c8022a00 x1876699507434496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761035.004206:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507434496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761035.004211:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761035.004213:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761035.004215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507434496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761035.004226:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507434624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761035.004237:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761035.004244:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761035/real 1789761035] req@ffff8d25c8020a80 x1876699507434624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761035.004252:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507434624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761035.004257:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761035.004259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761035.004261:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507434624:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761035.067502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761036.091530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761037.115552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761038.139589:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761039.163538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761040.123466:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761040.123477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761040.123481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761040.123490:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1017 00000100:00080000:2.0:1789761040.123499:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761040.123506:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761040.123548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507434752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761040.123568:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761040.123572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761040.123575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761040.123580:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1017 00000100:00080000:2.0:1789761040.123584:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761040.123589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761040.123599:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761040.123605:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507434752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761040.123611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507434880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761040.123622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761040.123625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761040.123626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761040.123630:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1017 00000100:00080000:2.0:1789761040.123634:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761040.123638:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761040.123643:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761040.123652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507435008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761040.123661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761040.123664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789761040.123664:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761040.123667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761040.123681:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1017 00000100:00080000:2.0:1789761040.123683:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761040.123687:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761040.123691:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761040.123700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507435136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761040.123709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761040.123712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761040.123714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761040.123717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1017 00000100:00080000:2.0:1789761040.123720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761040.123724:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761040.123727:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffff8d25c8022300 x1876699507434752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761040.123729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761040.123738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507435264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761040.123747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761040.123765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761040.123767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761040.123768:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507434752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761040.123772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1017 00000100:00080000:2.0:1789761040.123786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761040.123788:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761040.123789:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761040.123792:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789761040.123794:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761040.123796:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507434752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761040.123804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507435392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761040.123818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761040.123842:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761040.123855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffff8d25c8023100 x1876699507434880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761040.123864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507434880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761040.123877:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761040.123879:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761040.123881:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507434880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761040.123886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761040.123900:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761040.123908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffff8d25c8023b80 x1876699507435008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761040.123916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507435008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761040.123925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761040.123928:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761040.123930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761040.123933:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761040.123944:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761040.123950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffff8d25c8020700 x1876699507435136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761040.123957:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507435136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761040.123963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761040.123965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761040.123966:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761040.123969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507435264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761040.123985:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761040.123992:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffff8d25c8022d80 x1876699507435264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761040.124000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507435264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761040.124005:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761040.124007:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761040.124009:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507435264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761040.124013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507435392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761040.124024:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761040.124030:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761040/real 1789761040] req@ffff8d25c8021180 x1876699507435392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761040.124037:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507435392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761040.124042:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761040.124044:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761040.124046:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507435392:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761040.187574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761041.211502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761042.235542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761043.259503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761044.283629:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761045.243445:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.243454:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761045.243457:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761045.243467:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1022 00000100:00080000:2.0:1789761045.243477:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.243484:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761045.243517:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507435520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761045.243548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.243551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761045.243553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761045.243557:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1022 00000100:00080000:2.0:1789761045.243559:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.243563:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761045.243571:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761045.243582:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507435648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761045.243599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761045.243600:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507435520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761045.243603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761045.243608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761045.243625:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1022 00000100:00080000:2.0:1789761045.243629:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.243633:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761045.243638:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761045.243653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507435776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761045.243665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.243670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761045.243672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761045.243676:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1022 00000100:00080000:2.0:1789761045.243682:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.243686:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761045.243691:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761045.243696:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761045.243702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507435904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761045.243727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.243730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761045.243732:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761045.243737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1022 00000100:00080000:2.0:1789761045.243740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.243745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761045.243752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761045.243760:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffff8d25c8020000 x1876699507435520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761045.243770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507436032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761045.243782:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507435520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761045.243789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761045.243794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761045.243795:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761045.243798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761045.243802:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761045.243810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1022 00000100:00100000:3.0:1789761045.243811:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507435520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761045.243829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761045.243839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761045.243847:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761045.243853:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761045.243862:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507436160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761045.243879:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761045.243893:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffff8d25c8021500 x1876699507435648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761045.243902:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507435648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761045.243908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761045.243910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761045.243913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761045.243917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761045.243929:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761045.243935:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffff8d25c8023480 x1876699507435776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761045.243943:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507435776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761045.243948:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761045.243950:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761045.243952:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761045.243955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761045.243967:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761045.243974:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffff8d25c8021880 x1876699507435904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761045.243982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507435904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761045.243987:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761045.243989:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761045.243991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507435904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761045.243994:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507436032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761045.244005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761045.244011:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffff8d25c8021c00 x1876699507436032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761045.244026:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507436032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761045.244031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761045.244033:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761045.244036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507436032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761045.244047:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507436160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761045.244065:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761045.244077:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761045/real 1789761045] req@ffff8d25c8020e00 x1876699507436160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761045.244089:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507436160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761045.244105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761045.244109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761045.244113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507436160:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761045.307539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761046.331562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761047.355554:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:1.0:1789761048.379512:0:9279:0:(ldlm_lockd.c:2189:__ldlm_bl_to_thread()) added 3/3 locks to regular blp list, 1 blwis in pool 00010000:00010000:3.0:1789761048.379547:0:6393:0:(ldlm_lockd.c:2801:ldlm_bl_get_work()) Got 3 locks of 0 total in blp. (0 blwis in pool) 00010000:00010000:3.0:1789761048.379557:0:6393:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25c717be00/0x84ea57c0cd77077 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x28400000000 nid: local remote: 0x9638aab0164afc1f expref: -99 pid: 12700 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:2.0:1789761048.379575:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000080:00200000:3.0:1789761048.379585:0:6393:0:(vvp_object.c:117:vvp_conf_set()) [0x200000402:0x6:0x0]: losing layout lock 00010000:00010000:3.0:1789761048.379603:0:6393:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25c717b400/0x84ea57c0cd77070 lrc: 2/0,0 mode: PR/PR res: [0x200000402:0x4:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x9638aab0164afc03 expref: -99 pid: 12700 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789761048.379613:0:6393:0:(ldlm_request.c:1391:ldlm_cli_cancel_local()) ### client-side cancel ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25e195b400/0x84ea57c0cd7705b lrc: 2/0,0 mode: PR/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x8400000000 nid: local remote: 0x9638aab0164afba1 expref: -99 pid: 12693 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1789761048.379676:0:6393:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25c717be00/0x84ea57c0cd77077 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x9638aab0164afc1f expref: -99 pid: 12700 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789761048.379686:0:6393:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25c717b400/0x84ea57c0cd77070 lrc: 1/0,0 mode: --/PR res: [0x200000402:0x4:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x9638aab0164afc03 expref: -99 pid: 12700 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789761048.379726:0:6393:0:(ldlm_request.c:1439:__ldlm_pack_lock()) ### packing ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25e195b400/0x84ea57c0cd7705b lrc: 1/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x9638aab0164afba1 expref: -99 pid: 12693 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00010000:00010000:3.0:1789761048.379734:0:6393:0:(ldlm_request.c:1498:ldlm_cancel_pack()) 3 locks packed 00010000:00010000:3.0:1789761048.379759:0:6393:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25c717be00/0x84ea57c0cd77077 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x6:0x0].0x0 bits 0x1b/0x0 rrc: 2 type: IBT gid 0 flags: 0x4829400000020 nid: local remote: 0x9638aab0164afc1f expref: -99 pid: 12700 timeout: 0 lvb_type: 3 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789761048.379780:0:6393:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25c717b400/0x84ea57c0cd77070 lrc: 0/0,0 mode: --/PR res: [0x200000402:0x4:0x0].0x0 bits 0x11/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x9638aab0164afc03 expref: -99 pid: 12700 timeout: 0 lvb_type: 0 lru_score: 1 lru_type: 0 00010000:00010000:3.0:1789761048.379790:0:6393:0:(ldlm_lock.c:193:ldlm_lock_put()) ### final lock_put on destroyed lock, freeing it. ns: lustre-MDT0000-mdc-ffff8d25c5405000 lock: ffff8d25e195b400/0x84ea57c0cd7705b lrc: 0/0,0 mode: --/PR res: [0x200000007:0x1:0x0].0x0 bits 0x13/0x0 rrc: 2 type: IBT gid 0 flags: 0x4809400000020 nid: local remote: 0x9638aab0164afba1 expref: -99 pid: 12693 timeout: 0 lvb_type: 0 lru_score: 4 lru_type: 0 00000100:00080000:2.0:1789761048.379798:0:2418:0:(client.c:1845:ptlrpc_send_new_req()) @@@ req waiting for recovery: (FULL != DISCONN) req@ffff8d25d1b09f80 x1876699507436288/t0(0) o103->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:17/18 lens 336/224 e 0 to 0 dl 0 ref 1 fl Rpc:WQU/200/ffffffff rc 0/-1 job:'ldlm_bl.0' uid:0 gid:0 projid:4294967295 00010000:00010000:3.0:1789761048.379799:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761048.379802:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761048.379805:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761049.403539:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761050.363594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761050.363604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761050.363608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761050.363620:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1028 00000100:00080000:2.0:1789761050.363630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761050.363638:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761050.363706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507436416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761050.363736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761050.363740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761050.363743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761050.363748:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1028 00000100:00080000:2.0:1789761050.363752:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761050.363758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761050.363769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761050.363787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507436544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761050.363799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761050.363803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761050.363806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761050.363814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1028 00000100:00100000:3.0:1789761050.363818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507436416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761050.363820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761050.363832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761050.363838:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761050.363852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507436672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761050.363861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761050.363864:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761050.363866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761050.363869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1028 00000100:00080000:2.0:1789761050.363874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761050.363877:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761050.363882:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761050.363892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507436800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761050.363905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761050.363910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761050.363913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761050.363919:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1028 00000100:00080000:2.0:1789761050.363925:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761050.363930:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761050.363937:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761050.363952:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507436928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761050.363966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761050.363969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761050.363971:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761050.363975:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1028 00000800:00000100:3.0:1789761050.363992:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761050.364004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761050.364011:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761050.364018:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761050.364035:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507437056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761050.364185:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffff8d25c8020e00 x1876699507436416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761050.364256:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507436416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761050.364270:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761050.364277:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761050.364282:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507436416:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761050.364320:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507436544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761050.364342:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761050.364352:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffff8d25c8021c00 x1876699507436544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761050.364365:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507436544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761050.364371:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761050.364373:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761050.364412:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507436544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761050.364418:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507436672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761050.364436:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761050.364453:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffff8d25c8021880 x1876699507436672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761050.364461:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507436672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761050.364467:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761050.364469:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761050.364471:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507436672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761050.364475:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507436800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761050.364487:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761050.364493:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffff8d25c8023480 x1876699507436800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761050.364500:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507436800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761050.364510:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761050.364512:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761050.364514:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507436800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761050.364517:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507436928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761050.364528:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761050.364535:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffff8d25c8021500 x1876699507436928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761050.364542:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507436928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761050.364547:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761050.364548:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761050.364550:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507436928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761050.364554:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507437056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761050.364565:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761050.364571:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761050/real 1789761050] req@ffff8d25c8020000 x1876699507437056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761050.364577:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507437056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761075 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761050.364582:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761050.364584:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761050.364586:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507437056:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761050.427533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761051.451537:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761052.475606:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761053.499518:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761054.523514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761055.483480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761055.483488:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761055.483491:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761055.483500:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1033 00000100:00080000:0.0:1789761055.483507:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761055.483514:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761055.483546:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507437184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761055.483567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761055.483570:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761055.483571:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761055.483574:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1033 00000100:00080000:0.0:1789761055.483576:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761055.483579:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761055.483586:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761055.483595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507437312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761055.483602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761055.483607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761055.483608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761055.483608:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507437184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761055.483616:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1033 00000100:00080000:0.0:1789761055.483618:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761055.483621:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761055.483624:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761055.483633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507437440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761055.483654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761055.483656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761055.483658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761055.483661:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1033 00000100:00080000:0.0:1789761055.483664:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761055.483673:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761055.483677:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761055.483685:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761055.483692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507437568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761055.483704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761055.483708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761055.483710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761055.483714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1033 00000100:00080000:0.0:1789761055.483719:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761055.483724:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761055.483729:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761055/real 1789761055] req@ffff8d25d25b9f80 x1876699507437184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761055.483730:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761055.483745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507437696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761055.483750:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507437184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761055.483758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761055.483766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761055.483768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761055.483768:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761055.483773:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1033 00000100:00080000:3.0:1789761055.483776:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761055.483778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761055.483781:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507437184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761055.483785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761055.483790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761055.483797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507437824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761055.483807:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507437312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761055.483829:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761055.483839:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761055/real 1789761055] req@ffff8d25d25bb800 x1876699507437312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761055.483848:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507437312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761055.483854:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761055.483856:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761055.483858:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507437312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761055.483862:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507437440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761055.483873:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761055.483880:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761055/real 1789761055] req@ffff8d25d25b8380 x1876699507437440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761055.483887:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507437440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761055.483893:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761055.483894:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761055.483896:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507437440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761055.483900:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507437568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761055.483911:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761055.483917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761055/real 1789761055] req@ffff8d25d25b9180 x1876699507437568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761055.483924:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507437568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761055.483930:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761055.483931:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761055.483933:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507437568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761055.483937:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507437696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761055.483947:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761055.483953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761055/real 1789761055] req@ffff8d25d25ba680 x1876699507437696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761055.483965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507437696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761055.483970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761055.483972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761055.483974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507437696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761055.483985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507437824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761055.483995:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761055.484002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761055/real 1789761055] req@ffff8d25d25bbb80 x1876699507437824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761055.484010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507437824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761080 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761055.484015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761055.484016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761055.484018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507437824:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761055.547520:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761056.571522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761057.595605:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761058.619536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761059.643519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761060.603537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761060.603547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761060.603552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761060.603563:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1038 00000100:00080000:3.0:1789761060.603574:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761060.603584:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761060.603639:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507437952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761060.603673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761060.603677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761060.603680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761060.603685:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1038 00000100:00080000:3.0:1789761060.603689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761060.603694:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761060.603703:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761060.603717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507438080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761060.603724:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507437952:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761060.603729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761060.603733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761060.603735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761060.603738:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1038 00000100:00080000:3.0:1789761060.603741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761060.603743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761060.603747:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761060.603757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507438208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761060.603764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761060.603767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761060.603769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761060.603771:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1038 00000100:00080000:3.0:1789761060.603774:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761060.603777:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761060.603781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761060.603789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507438336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761060.603795:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761060.603801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761060.603804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761060.603806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761060.603810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1038 00000100:00080000:3.0:1789761060.603812:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761060.603816:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761060.603819:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761060.603825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507438464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761060.603831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761060.603833:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761060.603835:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761060.603838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1038 00000100:00080000:3.0:1789761060.603841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761060.603847:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761060.603848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761060/real 1789761060] req@ffff8d25d1b08e00 x1876699507437952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761060.603850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761060.603864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507438592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761060.603876:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507437952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761060.603890:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761060.603894:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761060.603899:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507437952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761060.603930:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761060.603957:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761060.603971:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761060/real 1789761060] req@ffff8d25d1b09880 x1876699507438080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761060.603985:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507438080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761060.603994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761060.603997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761060.604002:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761060.604008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761060.604030:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761060.604039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761060/real 1789761060] req@ffff8d25d1b0a680 x1876699507438208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761060.604047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507438208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761060.604052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761060.604055:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761060.604057:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761060.604060:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761060.604072:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761060.604078:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761060/real 1789761060] req@ffff8d25d1b0b800 x1876699507438336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761060.604085:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507438336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761060.604090:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761060.604092:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761060.604093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761060.604096:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507438464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761060.604106:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761060.604112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761060/real 1789761060] req@ffff8d25d1b08700 x1876699507438464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761060.604124:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507438464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761060.604129:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761060.604130:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761060.604132:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507438464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761060.604135:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507438592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761060.604145:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761060.604151:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761060/real 1789761060] req@ffff8d25d1b0bb80 x1876699507438592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761060.604158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507438592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761085 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761060.604163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761060.604164:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761060.604166:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507438592:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761060.667497:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761061.691530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761062.715516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761063.739511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761064.763516:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761065.723468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761065.723477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761065.723481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761065.723506:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1043 00000100:00080000:2.0:1789761065.723514:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761065.723521:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761065.723552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507438720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761065.723605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761065.723608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761065.723610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761065.723613:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1043 00000100:00080000:2.0:1789761065.723616:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761065.723618:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761065.723626:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761065.723637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507438848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761065.723646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761065.723650:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507438720:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761065.723650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761065.723655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761065.723660:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1043 00000100:00080000:2.0:1789761065.723662:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761065.723665:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761065.723668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761065.723677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507438976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761065.723683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761065.723692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761065.723693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761065.723697:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1043 00000100:00080000:2.0:1789761065.723699:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761065.723702:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761065.723705:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761065.723714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507439104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761065.723721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761065.723723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761065.723725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761065.723727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1043 00000100:00080000:2.0:1789761065.723729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761065.723731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761065.723734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761065.723743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507439232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761065.723769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761065.723773:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761065.723774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761065.723779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761065.723785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1043 00000100:00080000:2.0:1789761065.723789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761065.723793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761065.723798:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761065.723809:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761065.723811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507439360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761065.723835:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761065.723855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507439232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761065.723873:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507439360:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761065.787521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761066.811485:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761067.835516:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761068.795614:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761068.795650:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761068.795674:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761068.795719:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761068.795727:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761068.795733:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761068.795742:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761068.795748:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761068.795771:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761065/real 1789761068] req@ffff8d25c8023100 x1876699507439232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761068.795804:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507439232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761068.795816:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761068.795821:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761068.795826:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507439232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761068.795835:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761065/real 1789761068] req@ffff8d25c8022300 x1876699507439360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761068.795849:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507439360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761068.795857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761068.795868:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761068.795872:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507439360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761068.795879:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761065/real 1789761068] req@ffff8d25c8022d80 x1876699507438848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761068.795892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507438848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761068.795903:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761068.795906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761068.795910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761068.795917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761065/real 1789761068] req@ffff8d25c8020700 x1876699507438976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761068.795930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507438976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761068.795938:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761068.795942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761068.795945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507438976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761068.795952:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761065/real 1789761068] req@ffff8d25c8023b80 x1876699507439104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761068.795965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507439104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761068.795974:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761068.795977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761068.795981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761068.795988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761065/real 1789761068] req@ffff8d25c8021180 x1876699507438720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761068.796001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507438720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761090 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761068.796009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761068.796013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761068.796016:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507438720:192.168.203.141@tcp:250:kworker.0 00010000:00010000:2.0:1789761068.859493:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761069.883505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761070.843546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761070.843554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761070.843557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761070.843565:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1048 00000100:00080000:3.0:1789761070.843573:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761070.843579:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761070.843611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507439488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761070.843638:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761070.843643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761070.843645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761070.843659:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1048 00000100:00080000:3.0:1789761070.843664:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761070.843669:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761070.843679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761070.843686:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507439488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761070.843695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507439616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761070.843704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761070.843707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761070.843709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761070.843712:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1048 00000100:00080000:3.0:1789761070.843715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761070.843718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761070.843722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761070.843730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507439744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761070.843740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761070.843744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761070.843746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761070.843750:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1048 00000100:00080000:3.0:1789761070.843754:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761070.843761:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761070.843761:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761070.843771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761070.843784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507439872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761070.843795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761070.843804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761070.843807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761070.843812:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761070/real 1789761070] req@ffff8d25d1b09c00 x1876699507439488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761070.843815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1048 00000100:00080000:3.0:1789761070.843823:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761070.843832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761070.843838:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761070.843841:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507439488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761070.843856:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761070.843857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507440000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761070.843870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761070.843871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761070.843878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761070.843880:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507439488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761070.843882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761070.843892:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1048 00000100:00080000:3.0:1789761070.843896:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761070.843901:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761070.843906:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761070.843915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761070.843921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507440128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761070.843950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761070.843968:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761070/real 1789761070] req@ffff8d25d1b09180 x1876699507439616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761070.843982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507439616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761070.843991:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761070.843995:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761070.843999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761070.844006:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761070.844029:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761070.844041:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761070/real 1789761070] req@ffff8d25d1b0aa00 x1876699507439744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761070.844054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507439744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761070.844061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761070.844064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761070.844067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761070.844073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761070.844093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761070.844106:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761070/real 1789761070] req@ffff8d25d1b0b100 x1876699507439872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761070.844117:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507439872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761070.844125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761070.844129:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761070.844132:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507439872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761070.844138:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761070.844159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761070.844171:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761070/real 1789761070] req@ffff8d25d1b0ad80 x1876699507440000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761070.844182:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507440000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761070.844191:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761070.844194:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761070.844197:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761070.844220:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761070.844244:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761070.844257:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761070/real 1789761070] req@ffff8d25d1b08a80 x1876699507440128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761070.844269:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507440128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761095 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761070.844278:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761070.844281:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761070.844285:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440128:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761070.907508:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761071.931498:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761072.955539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761073.979517:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761075.003556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761075.963465:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761075.963474:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761075.963478:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761075.963501:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1053 00000100:00080000:2.0:1789761075.963513:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761075.963519:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761075.963552:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507440256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761075.963575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761075.963580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761075.963582:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761075.963585:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1053 00000100:00080000:2.0:1789761075.963588:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761075.963590:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761075.963597:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761075.963605:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507440384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761075.963612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761075.963614:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761075.963615:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761075.963617:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1053 00000100:00080000:2.0:1789761075.963619:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761075.963621:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761075.963624:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761075.963631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507440512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761075.963638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761075.963641:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761075.963643:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761075.963646:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1053 00000100:00080000:2.0:1789761075.963649:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761075.963650:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507440256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761075.963654:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761075.963661:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761075.963668:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507440640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761075.963675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761075.963677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761075.963678:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761075.963681:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1053 00000100:00080000:2.0:1789761075.963683:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761075.963685:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761075.963688:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761075.963695:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507440768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761075.963701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761075.963704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761075.963705:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761075.963708:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1053 00000800:00000100:0.0:1789761075.963712:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761075.963712:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761075.963718:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761075.963721:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761075.963729:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507440896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761075.963754:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761075/real 1789761075] req@ffff8d25c8020a80 x1876699507440256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761075.963768:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507440256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761075.963776:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761075.963779:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761075.963782:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507440256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761075.963786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507440384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761075.963800:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761075.963808:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761075/real 1789761075] req@ffff8d25c8022a00 x1876699507440384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761075.963816:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507440384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761075.963821:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761075.963823:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761075.963825:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507440384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761075.963829:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507440512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761075.963839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761075.963845:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761075/real 1789761075] req@ffff8d25c8022680 x1876699507440512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761075.963852:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507440512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761075.963858:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761075.963860:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761075.963861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507440512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761075.963880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507440640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761075.963891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761075.963898:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761075/real 1789761075] req@ffff8d25c8023800 x1876699507440640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761075.963905:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507440640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761075.963911:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761075.963913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761075.963914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507440640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761075.963918:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761075.963928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761075.963934:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761075/real 1789761075] req@ffff8d25c8020380 x1876699507440768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761075.963941:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507440768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761075.963946:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761075.963947:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761075.963949:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761075.963953:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761075.963972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761075.963980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761075/real 1789761075] req@ffff8d25c8021180 x1876699507440896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761075.963987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507440896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761100 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761075.963993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761075.963995:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761075.963997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507440896:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761076.027570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761077.051579:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761078.075500:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761079.099554:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761080.123513:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761081.083508:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761081.083519:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761081.083523:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.083534:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1058 00000100:00080000:0.0:1789761081.083544:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.083552:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761081.083586:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507441024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761081.083616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761081.083620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761081.083622:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.083626:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1058 00000100:00080000:0.0:1789761081.083629:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.083634:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761081.083644:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761081.083656:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507441024:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761081.083662:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507441152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761081.083676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761081.083680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761081.083682:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.083687:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1058 00000100:00080000:0.0:1789761081.083692:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.083696:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761081.083702:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761081.083713:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761081.083718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507441280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761081.083729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761081.083733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761081.083735:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.083739:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1058 00000100:00080000:0.0:1789761081.083743:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.083747:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761081.083752:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffff8d25d25b8000 x1876699507441024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761081.083754:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761081.083774:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507441024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761081.083778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507441408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761081.083790:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761081.083796:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761081.083797:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761081.083801:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507441024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761081.083806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761081.083812:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761081.083818:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1058 00000100:00080000:0.0:1789761081.083822:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761081.083826:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761081.083830:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441152:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789761081.083834:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761081.083851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507441536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761081.083855:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761081.083868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761081.083869:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffff8d25d25ba300 x1876699507441152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761081.083878:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761081.083883:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761081.083888:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507441152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761081.083891:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1058 00000100:00080000:0.0:1789761081.083898:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761081.083905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761081.083906:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761081.083916:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789761081.083917:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761081.083925:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761081.083934:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507441664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761081.083937:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761081.083968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761081.083980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffff8d25d25b8a80 x1876699507441280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761081.083988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507441280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761081.083994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761081.083996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761081.083999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761081.084002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761081.084013:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761081.084023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffff8d25d25bbb80 x1876699507441408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761081.084036:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507441408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761081.084045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761081.084049:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761081.084053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761081.084070:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507441536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761081.084086:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761081.084094:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffff8d25d25ba680 x1876699507441536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761081.084102:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507441536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761081.084108:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761081.084110:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761081.084112:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507441536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761081.084115:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507441664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761081.084125:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761081.084131:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761081/real 1789761081] req@ffff8d25d25b9180 x1876699507441664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761081.084138:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507441664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761106 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761081.084149:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761081.084151:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761081.084152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507441664:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761081.147542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761082.171515:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761083.195528:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761084.219584:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761085.243542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761086.203530:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.203538:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761086.203542:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.203550:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1063 00000100:00080000:3.0:1789761086.203560:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761086.203567:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761086.203604:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507441792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761086.203627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.203632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761086.203635:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.203640:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1063 00000100:00080000:3.0:1789761086.203645:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761086.203650:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761086.203659:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507441792:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789761086.203661:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761086.203683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507441920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761086.203697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.203703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761086.203705:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.203711:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1063 00000800:00000100:0.0:1789761086.203719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761086.203719:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761086.203730:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761086.203741:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761086.203762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507442048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761086.203776:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761086] req@ffff8d25d1b09500 x1876699507441792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761086.203777:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.203787:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761086.203793:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.203800:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1063 00000100:00080000:3.0:1789761086.203805:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761086.203811:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507441792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761086.203812:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761086.203823:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1789761086.203825:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761086.203831:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761086.203834:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507441792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761086.203854:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761086.203870:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761086.203879:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761086] req@ffff8d25d1b08000 x1876699507441920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761086.203887:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507441920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761086.203893:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761086.203895:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761086.203897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507441920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761086.203901:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761086.203911:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761086.203925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761086] req@ffff8d25d1b0b480 x1876699507442048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761086.203933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507442048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761086.203941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761086.203943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761086.203946:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761086.203955:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507442176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761086.203983:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.203987:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761086.203990:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.203996:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1063 00000100:00080000:3.0:1789761086.204002:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761086.204009:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761086.204016:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761086.204028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761086.204037:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507442304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761086.204056:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761086.204061:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761086.204064:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761086.204070:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1063 00000800:00000100:2.0:1789761086.204094:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761086.204118:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761086] req@ffff8d25d1b0a300 x1876699507442176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761086.204132:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507442176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761086.204140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761086.204143:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761086.204146:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761086.204168:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507442304:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761086.204230:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761086.204237:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761086.204242:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761086.204254:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761086.204256:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761086] req@ffff8d25d1b08380 x1876699507442304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761086.204278:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507442304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761086.204285:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761086.204287:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761086.204289:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507442304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761086.204324:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507442432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761086.204361:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507442432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761086.204428:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761086.204451:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761086/real 1789761086] req@ffff8d25d1b0a300 x1876699507442432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761086.204467:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507442432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761111 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761086.204473:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761086.204475:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761086.204478:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507442432:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761086.267575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761087.291550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761088.315519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761089.339530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761090.363533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761091.323459:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761091.323468:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761091.323472:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761091.323480:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1069 00000100:00080000:3.0:1789761091.323488:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761091.323495:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761091.323534:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507442560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761091.323562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761091.323567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761091.323570:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761091.323575:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1069 00000100:00080000:3.0:1789761091.323579:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761091.323585:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761091.323595:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761091.323601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507442560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761091.323610:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507442688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761091.323619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761091.323622:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761091.323623:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761091.323626:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1069 00000100:00080000:3.0:1789761091.323629:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761091.323632:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761091.323636:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761091.323643:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507442816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761091.323649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761091.323652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761091.323654:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761091.323656:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1069 00000100:00080000:3.0:1789761091.323658:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761091.323660:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761091.323663:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789761091.323671:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761091.323673:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507442944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761091.323681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761091.323685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761091.323686:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761091.323689:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1069 00000100:00080000:3.0:1789761091.323693:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761091.323698:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761091.323703:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761091.323717:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507443072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761091.323722:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffff8d25d1b0b480 x1876699507442560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761091.323737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761091.323743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761091.323745:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761091.323751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1069 00000100:00100000:2.0:1789761091.323752:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507442560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761091.323755:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761091.323762:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761091.323770:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789761091.323774:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761091.323782:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761091.323782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507443200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761091.323802:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507442560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761091.323830:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761091.323858:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761091.323872:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffff8d25d1b08000 x1876699507442688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761091.323887:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507442688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761091.323898:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761091.323902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761091.323906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761091.323913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761091.323935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761091.323947:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffff8d25d1b09500 x1876699507442816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761091.323960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507442816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761091.323969:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761091.323973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761091.323977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761091.323984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761091.324003:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761091.324016:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffff8d25d1b08a80 x1876699507442944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761091.324029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507442944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761091.324038:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761091.324042:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761091.324046:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507442944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761091.324053:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761091.324072:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761091.324083:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffff8d25d1b0ad80 x1876699507443072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761091.324096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507443072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761091.324105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761091.324109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761091.324113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761091.324119:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761091.324141:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761091.324153:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761091/real 1789761091] req@ffff8d25d1b0b100 x1876699507443200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761091.324167:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507443200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761116 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761091.324175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761091.324179:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761091.324182:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443200:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761091.387521:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761092.411559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761093.435560:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761094.459550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761095.483536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761096.443461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761096.443469:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761096.443475:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761096.443483:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1074 00000100:00080000:3.0:1789761096.443491:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761096.443496:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761096.443526:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507443328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761096.443548:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761096.443551:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761096.443552:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761096.443566:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1074 00000100:00080000:3.0:1789761096.443569:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761096.443571:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761096.443579:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761096.443588:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507443456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761096.443596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761096.443600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789761096.443602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507443328:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761096.443602:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761096.443608:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1074 00000100:00080000:3.0:1789761096.443613:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761096.443616:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761096.443619:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761096.443628:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507443584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761096.443640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761096.443643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761096.443644:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761096.443647:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1074 00000100:00080000:3.0:1789761096.443656:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761096.443659:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761096.443662:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789761096.443666:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761096.443673:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507443712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761096.443683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761096.443685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761096.443686:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761096.443689:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1074 00000100:00080000:3.0:1789761096.443691:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761096.443694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761096.443697:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761096.443703:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761096/real 1789761096] req@ffff8d25d1b0aa00 x1876699507443328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761096.443707:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507443840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761096.443729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761096.443733:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789761096.443734:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507443328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761096.443736:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761096.443744:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1074 00000100:00080000:3.0:1789761096.443750:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761096.443751:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761096.443752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761096.443759:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761096.443766:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507443968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761096.443767:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761096.443776:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507443328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761096.443800:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507443456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761096.443816:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761096.443825:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761096/real 1789761096] req@ffff8d25d1b09180 x1876699507443456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761096.443844:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507443456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761096.443851:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761096.443853:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761096.443855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507443456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761096.443859:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507443584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761096.443870:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761096.443876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761096/real 1789761096] req@ffff8d25d1b09c00 x1876699507443584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761096.443888:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507443584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761096.443894:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761096.443895:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761096.443897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507443584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761096.443901:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507443712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761096.443911:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761096.443917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761096/real 1789761096] req@ffff8d25d1b0bb80 x1876699507443712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761096.443923:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507443712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761096.443929:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761096.443931:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761096.443933:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507443712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761096.443936:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761096.443946:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761096.443952:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761096/real 1789761096] req@ffff8d25d1b08700 x1876699507443840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761096.443959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507443840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761096.443969:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761096.443972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761096.443973:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761096.443977:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761096.443986:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761096.443992:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761096/real 1789761096] req@ffff8d25d1b0b800 x1876699507443968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761096.443999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507443968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761121 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761096.444003:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761096.444005:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761096.444007:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507443968:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761096.507580:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761097.531555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761098.555537:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761099.579520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761100.603555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761101.563483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761101.563494:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761101.563499:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761101.563509:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1079 00000100:00080000:0.0:1789761101.563519:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761101.563527:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761101.563569:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507444096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761101.563593:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761101.563596:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761101.563597:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761101.563601:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1079 00000100:00080000:0.0:1789761101.563603:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761101.563606:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761101.563616:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761101.563626:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507444224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761101.563634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761101.563637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761101.563639:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761101.563643:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507444096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761101.563644:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1079 00000100:00080000:0.0:1789761101.563650:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761101.563654:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761101.563658:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761101.563666:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507444352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761101.563672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761101.563675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761101.563676:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761101.563679:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1079 00000100:00080000:0.0:1789761101.563681:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761101.563684:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761101.563687:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761101.563695:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507444480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761101.563702:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761101.563705:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761101.563706:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761101.563708:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1079 00000100:00080000:0.0:1789761101.563711:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761101.563713:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761101.563716:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761101.563724:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507444608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761101.563735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761101.563738:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761101.563739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761101.563743:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761101.563746:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1079 00000100:00080000:0.0:1789761101.563748:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761101.563751:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761101.563754:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761101.563762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507444736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761101.563801:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761101/real 1789761101] req@ffff8d25d25b8380 x1876699507444096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761101.563824:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507444096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761101.563837:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761101.563842:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761101.563847:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507444096:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761101.563876:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761101.563902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761101.563917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761101/real 1789761101] req@ffff8d25d25bb800 x1876699507444224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761101.563931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507444224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761101.563940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761101.563944:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761101.563948:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761101.563955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761101.563978:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761101.563991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761101/real 1789761101] req@ffff8d25d25b9f80 x1876699507444352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761101.564005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507444352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761101.564013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761101.564017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761101.564020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761101.564026:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761101.564048:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761101.564060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761101/real 1789761101] req@ffff8d25d25bad80 x1876699507444480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761101.564073:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507444480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761101.564081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761101.564085:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761101.564088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761101.564095:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507444608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761101.564117:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761101.564130:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761101/real 1789761101] req@ffff8d25d25b8700 x1876699507444608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761101.564142:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507444608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761101.564151:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761101.564154:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761101.564158:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507444608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761101.564164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507444736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761101.564187:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761101.564200:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761101/real 1789761101] req@ffff8d25d25bb100 x1876699507444736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761101.564220:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507444736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761126 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761101.564229:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761101.564232:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761101.564236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507444736:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761101.627544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761102.651613:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761103.675497:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761104.699557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761105.723522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761106.683483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761106.683497:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761106.683503:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761106.683516:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1084 00000100:00080000:3.0:1789761106.683529:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761106.683539:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761106.683586:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507444864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761106.683612:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761106.683615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761106.683616:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761106.683620:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1084 00000100:00080000:3.0:1789761106.683623:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761106.683627:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761106.683637:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761106.683649:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507444864:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761106.683649:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507444992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761106.683663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761106.683665:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761106.683667:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761106.683670:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1084 00000100:00080000:3.0:1789761106.683672:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761106.683675:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761106.683677:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761106.683685:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507445120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761106.683691:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761106.683694:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761106.683695:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761106.683697:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1084 00000100:00080000:3.0:1789761106.683700:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761106.683702:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761106.683705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761106.683714:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507445248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761106.683721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761106.683723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761106.683724:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761106.683726:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1084 00000100:00080000:3.0:1789761106.683729:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761106.683732:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761106.683735:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761106.683740:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761106.683744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507445376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761106.683751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761106.683755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761106.683757:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761106.683762:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1084 00000100:00080000:3.0:1789761106.683766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761106.683770:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761106.683775:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761106.683790:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507445504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761106.683795:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761106/real 1789761106] req@ffff8d25d1b0a680 x1876699507444864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761106.683829:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507444864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761106.683841:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761106.683846:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761106.683851:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507444864:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761106.683879:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761106.683899:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761106.683909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761106/real 1789761106] req@ffff8d25d1b09880 x1876699507444992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761106.683917:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507444992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761106.683933:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761106.683935:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761106.683938:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507444992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761106.683941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761106.683952:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761106.683958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761106/real 1789761106] req@ffff8d25d1b08e00 x1876699507445120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761106.683966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507445120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761106.683971:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761106.683973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761106.683975:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761106.683978:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761106.683989:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761106.683994:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761106/real 1789761106] req@ffff8d25d1b0b800 x1876699507445248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761106.684002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507445248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761106.684007:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761106.684008:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761106.684011:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761106.684015:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507445376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761106.684035:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761106.684041:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761106/real 1789761106] req@ffff8d25d1b08700 x1876699507445376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761106.684048:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507445376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761106.684053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761106.684054:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761106.684056:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507445376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761106.684059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507445504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761106.684069:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761106.684075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761106/real 1789761106] req@ffff8d25d1b0bb80 x1876699507445504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761106.684082:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507445504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761131 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761106.684096:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761106.684097:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761106.684099:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507445504:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761106.747548:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761107.771607:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761108.795542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761109.819550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761110.843567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761111.803520:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761111.803534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761111.803540:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761111.803554:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1089 00000100:00080000:3.0:1789761111.803566:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761111.803575:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761111.803623:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507445632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761111.803660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761111.803664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761111.803667:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761111.803671:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1089 00000100:00080000:3.0:1789761111.803676:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761111.803681:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761111.803694:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761111.803710:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507445632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761111.803710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507445760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761111.803736:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761111.803741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761111.803744:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761111.803750:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1089 00000100:00080000:3.0:1789761111.803755:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761111.803760:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761111.803766:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761111.803779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507445888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761111.803790:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761111.803793:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761111.803799:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761111.803802:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761111.803809:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1089 00000100:00080000:3.0:1789761111.803815:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761111.803830:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761111.803837:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761111/real 1789761111] req@ffff8d25d1b09c00 x1876699507445632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761111.803838:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761111.803879:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507446016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761111.803887:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507445632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761111.803897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761111.803901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761111.803904:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761111.803907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761111.803910:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761111.803914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507445632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761111.803921:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1089 00000100:00080000:3.0:1789761111.803928:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761111.803934:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761111.803940:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761111.803946:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761111.803957:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507446144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761111.803970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761111.803971:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761111.803978:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761111.803982:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761111.803987:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761111/real 1789761111] req@ffff8d25d1b09180 x1876699507445760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761111.803988:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1089 00000100:00080000:3.0:1789761111.804001:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761111.804007:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507445760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761111.804012:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789761111.804032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761111.804032:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789761111.804044:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761111.804048:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761111.804052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761111.804070:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507446272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761111.804079:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761111.804095:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761111/real 1789761111] req@ffff8d25d1b0aa00 x1876699507445888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761111.804145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507445888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761111.804153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761111.804156:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761111.804159:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507445888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761111.804164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761111.804177:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761111.804183:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761111/real 1789761111] req@ffff8d25d1b0b100 x1876699507446016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761111.804191:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507446016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761111.804197:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761111.804198:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761111.804200:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761111.804232:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507446144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761111.804248:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761111.804257:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761111/real 1789761111] req@ffff8d25d1b0ad80 x1876699507446144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761111.804264:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507446144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761111.804274:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761111.804277:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761111.804279:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507446144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761111.804283:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507446272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761111.804294:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761111.804300:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761111/real 1789761111] req@ffff8d25d1b08a80 x1876699507446272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761111.804307:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507446272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761136 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761111.804312:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761111.804314:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761111.804316:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507446272:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761111.867633:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761112.891512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761113.915505:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761114.939541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761115.963586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761116.923481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761116.923492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761116.923496:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761116.923506:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1094 00000100:00080000:2.0:1789761116.923515:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761116.923522:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761116.923554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507446400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761116.923584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761116.923588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761116.923590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761116.923594:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1094 00000100:00080000:2.0:1789761116.923598:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761116.923602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761116.923612:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761116.923625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507446528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761116.923633:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507446400:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761116.923636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761116.923639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761116.923641:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761116.923645:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1094 00000100:00080000:2.0:1789761116.923648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761116.923652:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761116.923656:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761116.923686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507446656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761116.923695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761116.923698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761116.923700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761116.923702:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761116.923704:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1094 00000100:00080000:2.0:1789761116.923707:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761116.923710:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761116.923715:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761116.923725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507446784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761116.923734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761116.923737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761116.923738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761116.923742:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761116/real 1789761116] req@ffff8d25c8023b80 x1876699507446400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761116.923742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1094 00000100:00080000:2.0:1789761116.923745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761116.923749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761116.923754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761116.923759:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507446400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761116.923763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507446912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761116.923770:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761116.923772:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761116.923774:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761116.923776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761116.923778:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507446400:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761116.923778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761116.923781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1094 00000100:00080000:2.0:1789761116.923795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761116.923799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761116.923803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761116.923812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507447040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761116.923813:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761116.923834:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761116.923848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761116/real 1789761116] req@ffff8d25c8020700 x1876699507446528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761116.923862:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507446528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761116.923870:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761116.923873:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761116.923875:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761116.923888:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761116.923904:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761116.923912:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761116/real 1789761116] req@ffff8d25c8022d80 x1876699507446656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761116.923920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507446656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761116.923925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761116.923927:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761116.923929:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761116.923932:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761116.923942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761116.923948:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761116/real 1789761116] req@ffff8d25c8022300 x1876699507446784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761116.923956:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507446784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761116.923961:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761116.923972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761116.923974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507446784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761116.923977:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507446912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761116.923987:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761116.923994:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761116/real 1789761116] req@ffff8d25c8023100 x1876699507446912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761116.924001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507446912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761116.924007:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761116.924010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761116.924013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507446912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761116.924030:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507447040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761116.924051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761116.924063:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761116/real 1789761116] req@ffff8d25c8020000 x1876699507447040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761116.924072:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507447040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761141 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761116.924082:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761116.924084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761116.924086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507447040:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761116.987525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761118.011527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761119.035507:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761120.059562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761121.083522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761122.043529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761122.043543:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761122.043549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761122.043562:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1099 00000100:00080000:0.0:1789761122.043574:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761122.043587:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761122.043640:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507447168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761122.043676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761122.043680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761122.043682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761122.043687:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1099 00000100:00080000:0.0:1789761122.043690:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761122.043695:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761122.043707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761122.043723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507447296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761122.043730:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507447168:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761122.043742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761122.043748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761122.043751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761122.043756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1099 00000100:00080000:0.0:1789761122.043760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761122.043764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761122.043770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761122.043808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507447424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761122.043822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761122.043827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761122.043829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761122.043833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1099 00000100:00080000:0.0:1789761122.043855:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761122.043855:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761122.043862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761122.043868:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761122.043881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507447552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761122.043892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761122.043896:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761122.043898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761122.043902:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1099 00000100:00080000:0.0:1789761122.043907:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761122.043908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffff8d25d25b9c00 x1876699507447168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761122.043916:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761122.043926:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761122.043930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507447168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761122.043941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789761122.043943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507447680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761122.043948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761122.043955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507447168:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761122.043964:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761122.043969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761122.043971:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761122.043977:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1099 00000100:00080000:0.0:1789761122.043981:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761122.043986:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761122.043986:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507447296:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789761122.043995:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761122.044024:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507447808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761122.044035:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761122.044050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffff8d25d25b8e00 x1876699507447296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761122.044060:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507447296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761122.044068:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761122.044071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761122.044074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507447296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761122.044094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507447424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761122.044108:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761122.044115:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffff8d25d25bb480 x1876699507447424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761122.044123:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507447424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761122.044129:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761122.044131:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761122.044133:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507447424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761122.044136:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507447552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761122.044169:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761122.044179:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffff8d25d25bb100 x1876699507447552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761122.044203:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507447552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761122.044213:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761122.044217:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761122.044222:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507447552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761122.044228:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507447680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761122.044249:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761122.044258:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffff8d25d25b8700 x1876699507447680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761122.044275:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507447680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761122.044281:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761122.044284:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761122.044286:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507447680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761122.044306:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507447808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761122.044324:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761122.044337:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761122/real 1789761122] req@ffff8d25d25bad80 x1876699507447808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761122.044357:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507447808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761147 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761122.044363:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761122.044365:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761122.044367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507447808:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761122.107550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761123.131503:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761124.155589:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761125.179529:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761126.203603:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761127.163521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.163537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761127.163542:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.163554:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1104 00000100:00080000:0.0:1789761127.163566:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.163574:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761127.163621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507447936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761127.163660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.163665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761127.163668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.163679:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1104 00000100:00080000:0.0:1789761127.163685:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.163691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761127.163704:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761127.163715:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507447936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761127.163723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507448064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761127.163734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.163738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761127.163740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.163745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1104 00000100:00080000:0.0:1789761127.163750:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.163755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761127.163761:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761127.163776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507448192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761127.163786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.163791:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1789761127.163792:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761127.163794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.163801:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1104 00000100:00080000:0.0:1789761127.163804:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.163807:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761127.163812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761127.163822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507448320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761127.163830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.163832:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761127.163833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.163836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1104 00000100:00080000:0.0:1789761127.163838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.163840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761127.163843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761127.163851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507448448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761127.163857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761127.163860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761127.163862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761127.163867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1104 00000100:00100000:2.0:1789761127.163868:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffff8d25d25b9f80 x1876699507447936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761127.163869:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761127.163874:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761127.163880:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761127.163890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507448576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761127.163914:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507447936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761127.163928:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761127.163932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761127.163938:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507447936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761127.163974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761127.164002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761127.164023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffff8d25d25bb800 x1876699507448064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761127.164035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507448064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761127.164044:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761127.164047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761127.164050:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761127.164057:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761127.164077:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761127.164089:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffff8d25d25b8380 x1876699507448192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761127.164119:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507448192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761127.164125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761127.164127:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761127.164129:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761127.164133:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761127.164150:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761127.164157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffff8d25d25b9180 x1876699507448320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761127.164165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507448320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761127.164170:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761127.164172:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761127.164174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761127.164177:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507448448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761127.164188:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761127.164194:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffff8d25d25ba680 x1876699507448448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761127.164202:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507448448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761127.164207:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761127.164208:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761127.164210:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507448448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761127.164213:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507448576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761127.164223:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761127.164229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761127/real 1789761127] req@ffff8d25d25bbb80 x1876699507448576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761127.164237:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507448576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761152 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761127.164242:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761127.164243:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761127.164245:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507448576:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761127.227591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761128.251557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761129.275538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761130.299563:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761131.323563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761132.283472:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761132.283482:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761132.283487:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761132.283496:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1109 00000100:00080000:1.0:1789761132.283506:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761132.283513:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761132.283551:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507448704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761132.283581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761132.283586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761132.283588:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761132.283593:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1109 00000100:00080000:1.0:1789761132.283596:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761132.283600:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761132.283611:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761132.283631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507448704:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761132.283632:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507448832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761132.283651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761132.283655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761132.283658:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761132.283663:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1109 00000100:00080000:1.0:1789761132.283667:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761132.283672:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761132.283679:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761132.283694:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507448960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761132.283704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761132.283708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761132.283710:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761132.283715:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1109 00000100:00080000:1.0:1789761132.283720:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761132.283725:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761132.283732:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761132.283735:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761132.283751:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507449088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761132.283766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761132.283768:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448960:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761132.283774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761132.283779:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761132.283785:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1109 00000100:00080000:1.0:1789761132.283791:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761132.283793:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449088:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1789761132.283799:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761132.283807:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761132.283825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507449216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761132.283844:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761132.283848:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761132.283851:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761132.283855:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1109 00000100:00080000:1.0:1789761132.283860:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761132.283865:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761132.283869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507449216:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1789761132.283873:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761132.283890:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507449344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761132.283910:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507449344:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761132.347530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761133.371498:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761134.395514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761135.355849:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1789761135.355894:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761135.355908:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761135.355943:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761135.355950:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761135.355956:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761135.355962:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761135.355967:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761135.356032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761135] req@ffff8d25c8168380 x1876699507449344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761135.356082:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507449344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761135.356095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761135.356100:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761135.356104:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507449344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761135.356111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761135] req@ffff8d25c816b480 x1876699507449216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761135.356123:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507449216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761135.356130:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761135.356133:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761135.356136:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507449216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761135.356142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761135] req@ffff8d25c816a300 x1876699507449088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761135.356153:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507449088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761135.356161:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761135.356165:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761135.356168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761135.356175:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761135] req@ffff8d25c816ad80 x1876699507448832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761135.356188:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507448832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761135.356196:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761135.356199:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761135.356203:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761135.356209:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761135] req@ffff8d25c8168a80 x1876699507448960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761135.356222:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507448960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761135.356243:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761135.356247:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761135.356251:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507448960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761135.356258:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761132/real 1789761135] req@ffff8d25c8169880 x1876699507448704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761135.356273:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507448704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761157 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761135.356281:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761135.356285:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761135.356289:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507448704:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789761135.419535:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761136.443514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761137.403461:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761137.403470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761137.403473:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761137.403482:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1115 00000100:00080000:1.0:1789761137.403491:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761137.403498:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761137.403526:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507449472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761137.403550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761137.403554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761137.403555:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761137.403558:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1115 00000100:00080000:1.0:1789761137.403560:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761137.403564:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761137.403571:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761137.403590:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507449600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761137.403598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761137.403600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761137.403602:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761137.403605:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1115 00000100:00080000:1.0:1789761137.403607:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761137.403612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507449472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761137.403615:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761137.403625:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761137.403645:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507449728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761137.403658:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761137.403662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761137.403663:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761137.403667:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1115 00000100:00080000:1.0:1789761137.403672:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761137.403675:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761137.403680:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789761137.403682:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761137.403698:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507449856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761137.403716:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761137.403721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761137.403723:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761137.403731:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1115 00000100:00100000:0.0:1789761137.403732:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761137/real 1789761137] req@ffff8d25c8169c00 x1876699507449472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761137.403737:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761137.403746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761137.403755:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761137.403762:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507449472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761137.403771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507449984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761137.403780:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761137.403787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761137.403789:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789761137.403796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761137.403799:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507449472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761137.403801:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761137.403811:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1115 00000100:00100000:0.0:1789761137.403815:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761137.403818:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761137.403831:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761137.403838:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761137.403852:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761137.403855:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507450112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761137.403873:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761137/real 1789761137] req@ffff8d25c8169880 x1876699507449600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761137.403888:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507449600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761137.403897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761137.403901:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761137.403904:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761137.403930:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761137.403951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761137.403963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761137/real 1789761137] req@ffff8d25c8168a80 x1876699507449728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761137.403981:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507449728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761137.403988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761137.403992:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761137.403995:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761137.404001:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761137.404017:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761137.404046:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761137/real 1789761137] req@ffff8d25c816ad80 x1876699507449856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761137.404059:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507449856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761137.404067:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761137.404071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761137.404074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507449856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761137.404080:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507449984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761137.404099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761137.404110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761137/real 1789761137] req@ffff8d25c816a300 x1876699507449984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761137.404121:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507449984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761137.404141:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761137.404144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761137.404147:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507449984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761137.404153:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507450112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761137.404172:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761137.404184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761137/real 1789761137] req@ffff8d25c816b480 x1876699507450112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761137.404195:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507450112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761162 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761137.404203:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761137.404206:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761137.404209:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507450112:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761137.467549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761138.491497:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761139.515585:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761140.539540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761141.563589:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761142.523517:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761142.523527:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761142.523532:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761142.523542:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1120 00000100:00080000:3.0:1789761142.523553:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761142.523561:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761142.523602:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507450240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761142.523627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761142.523630:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761142.523631:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761142.523635:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1120 00000100:00080000:3.0:1789761142.523637:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761142.523640:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761142.523649:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761142.523661:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507450368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761142.523671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761142.523675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761142.523677:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761142.523682:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1120 00000100:00080000:3.0:1789761142.523685:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761142.523690:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507450240:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761142.523694:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761142.523703:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761142.523718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507450496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761142.523730:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761142.523734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761142.523736:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761142.523740:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1120 00000100:00080000:3.0:1789761142.523745:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761142.523748:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761142.523753:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761142.523765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507450624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761142.523769:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761142.523786:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761142.523789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761142.523790:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761142.523794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1120 00000100:00080000:3.0:1789761142.523798:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761142.523800:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761142.523804:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761142.523812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507450752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761142.523819:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761142.523823:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761142/real 1789761142] req@ffff8d25d1b09500 x1876699507450240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761142.523824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761142.523827:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761142.523833:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1120 00000100:00080000:3.0:1789761142.523837:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761142.523842:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761142.523846:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761142.523853:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507450240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761142.523857:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507450880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761142.523870:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761142.523882:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761142.523886:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507450240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761142.523894:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507450368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761142.523913:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761142.523922:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761142/real 1789761142] req@ffff8d25d1b08000 x1876699507450368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761142.523931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507450368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761142.523937:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761142.523939:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761142.523942:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507450368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761142.523969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507450496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761142.523980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761142.524002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761142/real 1789761142] req@ffff8d25d1b0b480 x1876699507450496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761142.524011:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507450496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761142.524022:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761142.524024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761142.524027:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507450496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761142.524031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507450624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761142.524046:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761142.524054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761142/real 1789761142] req@ffff8d25d1b0a300 x1876699507450624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761142.524062:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507450624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761142.524067:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761142.524069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761142.524071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507450624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761142.524074:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507450752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761142.524086:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761142.524092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761142/real 1789761142] req@ffff8d25d1b08380 x1876699507450752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761142.524100:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507450752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761142.524105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761142.524106:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761142.524108:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507450752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761142.524111:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507450880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761142.524122:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761142.524128:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761142/real 1789761142] req@ffff8d25d1b08a80 x1876699507450880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761142.524141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507450880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761167 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761142.524146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761142.524148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761142.524150:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507450880:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761142.587530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761143.611536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761144.635522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761145.659585:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761146.683602:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761147.643512:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761147.643522:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761147.643527:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761147.643548:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1125 00000100:00080000:1.0:1789761147.643561:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761147.643572:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761147.643618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507451008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761147.643644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761147.643647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761147.643649:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761147.643652:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1125 00000100:00080000:1.0:1789761147.643655:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761147.643658:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761147.643669:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761147.643680:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507451136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761147.643688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761147.643692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761147.643693:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761147.643700:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1125 00000100:00080000:1.0:1789761147.643704:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761147.643711:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507451008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761147.643712:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761147.643728:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761147.643747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507451264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761147.643759:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761147.643762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761147.643763:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761147.643766:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1125 00000100:00080000:1.0:1789761147.643773:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761147.643777:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761147.643782:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789761147.643783:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761147.643795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507451392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761147.643805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761147.643808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761147.643809:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761147.643812:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1125 00000100:00080000:1.0:1789761147.643816:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761147.643819:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761147.643825:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761147.643830:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761147/real 1789761147] req@ffff8d25c8168380 x1876699507451008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761147.643843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507451520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761147.643848:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507451008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761147.643862:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761147.643876:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761147.643882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507451008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761147.643897:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761147.643903:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761147.643905:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761147.643911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451136:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761147.643913:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1125 00000100:00080000:1.0:1789761147.643929:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761147.643938:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761147.643943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789761147.643945:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761147.643975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761147/real 1789761147] req@ffff8d25c8168000 x1876699507451136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761147.643981:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507451648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761147.643998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507451136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761147.644116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761147.644123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761147.644128:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761147.644135:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761147.644156:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761147.644165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761147/real 1789761147] req@ffff8d25c8169500 x1876699507451264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761147.644174:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507451264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761147.644180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761147.644182:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761147.644184:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761147.644188:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761147.644200:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761147.644206:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761147/real 1789761147] req@ffff8d25c816aa00 x1876699507451392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761147.644213:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507451392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761147.644219:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761147.644221:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761147.644223:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761147.644227:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507451520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761147.644238:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761147.644245:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761147/real 1789761147] req@ffff8d25c816a680 x1876699507451520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761147.644253:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507451520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761147.644258:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761147.644259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761147.644261:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507451520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761147.644276:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507451648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761147.644287:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761147.644294:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761147/real 1789761147] req@ffff8d25c8169f80 x1876699507451648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761147.644302:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507451648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761172 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761147.644307:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761147.644308:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761147.644310:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507451648:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761147.707565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761148.731538:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761149.755522:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761150.779588:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761151.803589:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761152.763504:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761152.763513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761152.763517:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761152.763528:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1130 00000100:00080000:1.0:1789761152.763536:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761152.763542:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761152.763575:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507451776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761152.763597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761152.763600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761152.763602:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761152.763605:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1130 00000100:00080000:1.0:1789761152.763607:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761152.763610:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761152.763617:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761152.763626:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507451904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761152.763631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507451776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761152.763640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761152.763643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761152.763644:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761152.763647:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1130 00000100:00080000:1.0:1789761152.763649:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761152.763651:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761152.763655:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761152.763663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507452032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761152.763670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761152.763673:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761152.763674:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761152.763676:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1130 00000100:00080000:1.0:1789761152.763685:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761152.763687:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761152.763692:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789761152.763693:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761152.763702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507452160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761152.763710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761152.763712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761152.763713:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761152.763716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1130 00000100:00080000:1.0:1789761152.763718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761152.763720:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761152.763724:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761152.763731:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761152/real 1789761152] req@ffff8d25c8168e00 x1876699507451776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761152.763734:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507452288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761152.763761:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761152.763764:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761152.763765:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761152.763768:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507451776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761152.763771:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1130 00000100:00080000:1.0:1789761152.763777:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761152.763784:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761152.763787:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789761152.763788:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761152.763795:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761152.763800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507452416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761152.763801:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507451776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761152.763829:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761152.763846:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761152.763855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761152/real 1789761152] req@ffff8d25c8168700 x1876699507451904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761152.763863:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507451904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761152.763869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761152.763871:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761152.763873:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507451904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761152.763876:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761152.763886:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761152.763893:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761152/real 1789761152] req@ffff8d25c8169f80 x1876699507452032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761152.763905:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507452032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761152.763925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761152.763928:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761152.763930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761152.763934:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761152.763949:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761152.763957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761152/real 1789761152] req@ffff8d25c816a680 x1876699507452160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761152.763974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507452160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761152.763983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761152.763987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761152.763991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761152.763997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507452288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761152.764017:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761152.764027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761152/real 1789761152] req@ffff8d25c816aa00 x1876699507452288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761152.764035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507452288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761152.764044:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761152.764046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761152.764048:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507452288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761152.764052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507452416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761152.764063:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761152.764070:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761152/real 1789761152] req@ffff8d25c8169500 x1876699507452416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761152.764077:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507452416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761177 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761152.764081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761152.764083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761152.764085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507452416:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761152.827556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761153.851559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761154.875562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761155.899517:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761156.923552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761157.883496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761157.883507:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761157.883511:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761157.883520:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1135 00000100:00080000:1.0:1789761157.883527:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761157.883532:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761157.883563:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507452544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761157.883584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761157.883587:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761157.883588:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761157.883591:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1135 00000100:00080000:1.0:1789761157.883594:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761157.883598:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761157.883605:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761157.883615:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507452544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761157.883616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507452672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761157.883634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761157.883637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761157.883638:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761157.883641:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1135 00000100:00080000:1.0:1789761157.883643:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761157.883646:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761157.883649:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761157.883657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507452800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761157.883663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761157.883666:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761157.883667:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761157.883670:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1135 00000100:00080000:1.0:1789761157.883673:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761157.883674:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761157.883680:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761157.883684:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761157.883692:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507452928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761157.883698:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761157.883700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761157.883701:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761157.883704:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1135 00000100:00080000:1.0:1789761157.883707:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761157.883710:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761157.883714:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761157.883718:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761157/real 1789761157] req@ffff8d25c8168000 x1876699507452544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761157.883725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507453056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761157.883740:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761157.883743:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761157.883746:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761157.883747:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507452544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761157.883751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1135 00000100:00080000:1.0:1789761157.883756:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761157.883763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761157.883765:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789761157.883767:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761157.883771:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761157.883776:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507452544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761157.883779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507453184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761157.883801:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761157.883821:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761157.883833:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761157/real 1789761157] req@ffff8d25c8168380 x1876699507452672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761157.883843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507452672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761157.883849:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761157.883851:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761157.883856:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761157.883860:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761157.883894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761157.883905:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761157/real 1789761157] req@ffff8d25c816b480 x1876699507452800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761157.883916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507452800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761157.883923:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761157.883925:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761157.883927:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761157.883932:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761157.883956:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761157.883969:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761157/real 1789761157] req@ffff8d25c816a300 x1876699507452928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761157.883979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507452928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761157.883985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761157.883987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761157.883990:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507452928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761157.883994:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761157.884007:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761157.884014:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761157/real 1789761157] req@ffff8d25c816ad80 x1876699507453056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761157.884023:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507453056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761157.884029:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761157.884031:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761157.884033:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761157.884037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761157.884048:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761157.884056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761157/real 1789761157] req@ffff8d25c8168a80 x1876699507453184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761157.884069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507453184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761182 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761157.884075:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761157.884077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761157.884080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453184:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761157.947516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761158.971542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761159.995557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761161.019591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761162.043555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761163.003450:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.003458:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761163.003462:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.003471:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1140 00000100:00080000:1.0:1789761163.003478:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.003484:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761163.003514:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507453312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761163.003536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.003539:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761163.003541:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.003544:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1140 00000100:00080000:1.0:1789761163.003546:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.003550:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761163.003558:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761163.003567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507453440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761163.003572:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507453312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761163.003579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.003583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761163.003584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.003587:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1140 00000100:00080000:1.0:1789761163.003589:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.003592:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761163.003595:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761163.003605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507453568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761163.003611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.003613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761163.003615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.003617:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1140 00000100:00080000:1.0:1789761163.003621:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.003623:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761163.003628:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789761163.003632:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761163.003641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507453696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761163.003649:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.003651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761163.003652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761163.003654:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1140 00000100:00080000:1.0:1789761163.003657:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761163.003659:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761163.003663:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761163.003670:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffff8d25c8169880 x1876699507453312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761163.003673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507453824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761163.003695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761163.003698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761163.003699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761163.003701:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507453312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761163.003707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1140 00000100:00080000:0.0:1789761163.003714:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761163.003716:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761163.003719:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761163.003726:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507453312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761163.003726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761163.003738:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761163.003755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507453952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761163.003761:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507453440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761163.003787:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761163.003799:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffff8d25c8169c00 x1876699507453440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.003808:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507453440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.003818:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.003820:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761163.003823:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507453440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761163.003826:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507453568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761163.003851:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761163.003859:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffff8d25c8168a80 x1876699507453568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.003867:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507453568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.003872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.003875:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761163.003877:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507453568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761163.003880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507453696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761163.003891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761163.003903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffff8d25c816ad80 x1876699507453696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.003911:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507453696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.003916:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.003918:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761163.003921:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507453696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761163.003925:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761163.003936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761163.003943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffff8d25c816a300 x1876699507453824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.003950:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507453824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.003955:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.003957:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761163.003959:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761163.003971:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761163.003982:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761163.003988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761163/real 1789761163] req@ffff8d25c816b480 x1876699507453952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761163.003996:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507453952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761188 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761163.004001:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761163.004003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761163.004004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507453952:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761163.067550:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761164.091524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761165.115555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761166.139543:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761167.163541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761168.123471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.123478:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761168.123483:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.123491:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1145 00000100:00080000:1.0:1789761168.123500:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.123506:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761168.123536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507454080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761168.123559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.123563:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761168.123565:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.123571:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1145 00000100:00080000:1.0:1789761168.123575:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.123581:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761168.123591:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761168.123602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507454080:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761168.123609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507454208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761168.123622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.123626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761168.123629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.123633:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1145 00000100:00080000:1.0:1789761168.123639:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.123644:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761168.123649:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761168.123665:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761168.123666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507454336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761168.123686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.123690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761168.123692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.123697:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1145 00000100:00100000:0.0:1789761168.123702:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffff8d25c8168380 x1876699507454080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761168.123702:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.123711:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761168.123720:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761168.123723:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507454080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.123732:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.123736:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761168.123737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507454464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761168.123742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507454080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761168.123759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.123764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761168.123766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.123771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1145 00000100:00100000:0.0:1789761168.123776:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761168.123777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.123786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761168.123806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761168.123821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507454592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761168.123825:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761168.123839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761168.123843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761168.123846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffff8d25c8168000 x1876699507454208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761168.123847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761168.123856:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1145 00000100:00080000:1.0:1789761168.123864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761168.123871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761168.123874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507454208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761168.123879:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761168.123896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507454720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761168.123900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.123912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761168.123921:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761168.123929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761168.123948:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761168.123958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffff8d25c8169500 x1876699507454336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761168.123971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507454336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.123977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.123979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761168.123981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761168.123984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761168.123996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761168.124002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffff8d25c816aa00 x1876699507454464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761168.124009:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507454464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.124015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.124017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761168.124018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761168.124031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507454592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761168.124042:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761168.124048:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffff8d25c816a680 x1876699507454592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761168.124056:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507454592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.124065:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.124067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761168.124069:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507454592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761168.124072:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507454720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761168.124081:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761168.124087:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761168/real 1789761168] req@ffff8d25c8169f80 x1876699507454720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761168.124094:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507454720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761193 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761168.124099:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761168.124101:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761168.124102:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507454720:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761168.187550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761169.211534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761170.235537:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761171.259521:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761172.283546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761173.243478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761173.243487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761173.243492:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761173.243502:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1150 00000100:00080000:1.0:1789761173.243513:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761173.243524:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761173.243558:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507454848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761173.243583:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761173.243586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761173.243587:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761173.243590:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1150 00000100:00080000:1.0:1789761173.243593:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761173.243598:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761173.243606:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761173.243616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507454976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761173.243627:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761173.243628:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507454848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761173.243632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761173.243638:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761173.243645:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1150 00000100:00080000:1.0:1789761173.243647:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761173.243649:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761173.243652:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761173.243665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507455104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761173.243672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761173.243674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761173.243676:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761173.243678:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1150 00000100:00080000:1.0:1789761173.243681:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761173.243683:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761173.243686:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761173.243692:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507455232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761173.243700:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761173.243703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761173.243704:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761173.243706:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1150 00000100:00080000:1.0:1789761173.243709:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761173.243711:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761173.243719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789761173.243720:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761173.243735:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507455360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761173.243744:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761173.243747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761173.243748:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761173.243751:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1150 00000100:00080000:1.0:1789761173.243766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761173.243771:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761173.243776:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761173.243789:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507455488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761173.243801:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffff8d25c8168700 x1876699507454848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761173.243826:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507454848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761173.243839:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761173.243845:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761173.243850:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507454848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761173.243880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761173.243902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761173.243914:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffff8d25c8168e00 x1876699507454976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761173.243923:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507454976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761173.243929:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761173.243931:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761173.243934:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507454976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761173.243938:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761173.243950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761173.243957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffff8d25c8169f80 x1876699507455104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761173.243965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507455104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761173.243970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761173.243972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761173.243974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761173.243979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761173.243991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761173.243997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffff8d25c816a680 x1876699507455232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761173.244005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507455232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761173.244010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761173.244012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761173.244014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761173.244018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507455360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761173.244029:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761173.244035:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffff8d25c816aa00 x1876699507455360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761173.244043:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507455360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761173.244048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761173.244050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761173.244053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507455360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761173.244059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507455488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761173.244079:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761173.244091:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761173/real 1789761173] req@ffff8d25c8169500 x1876699507455488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761173.244111:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507455488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761198 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761173.244117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761173.244119:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761173.244121:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507455488:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761173.307514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761174.331550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761175.355494:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761176.379547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761177.403531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761178.363503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761178.363513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761178.363517:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761178.363527:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1156 00000100:00080000:2.0:1789761178.363538:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761178.363545:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761178.363585:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507455616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761178.363611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761178.363615:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761178.363617:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761178.363622:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1156 00000100:00080000:2.0:1789761178.363626:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761178.363631:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761178.363641:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761178.363657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507455744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761178.363658:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507455616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761178.363676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761178.363680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761178.363682:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761178.363692:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1156 00000100:00080000:2.0:1789761178.363696:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761178.363701:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761178.363706:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761178.363731:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507455872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761178.363743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761178.363749:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761178.363750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761178.363754:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761178.363760:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1156 00000100:00080000:2.0:1789761178.363765:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761178.363771:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761178.363777:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761178.363791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507456000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761178.363798:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffff8d25c8021500 x1876699507455616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761178.363838:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761178.363843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507455616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761178.363844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761178.363848:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761178.363857:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1156 00000100:00080000:0.0:1789761178.363861:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761178.363862:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761178.363870:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761178.363871:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761178.363878:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507455616:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761178.363878:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761178.363894:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507456128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761178.363903:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761178.363905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455744:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761178.363907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761178.363912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761178.363915:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1156 00000100:00080000:2.0:1789761178.363918:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761178.363921:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761178.363925:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761178.363933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761178.363935:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507456256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761178.363948:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffff8d25c8023480 x1876699507455744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.363960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507455744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.363970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.363972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761178.363975:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761178.363979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761178.363990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761178.363997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffff8d25c8021880 x1876699507455872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.364004:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507455872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.364009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.364011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761178.364013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507455872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761178.364016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761178.364026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761178.364032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffff8d25c8021c00 x1876699507456000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.364040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507456000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.364044:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.364046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761178.364048:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761178.364059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507456128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761178.364069:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761178.364075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffff8d25c8020e00 x1876699507456128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.364082:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507456128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.364087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.364089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761178.364091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507456128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761178.364094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507456256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761178.364104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761178.364110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761178/real 1789761178] req@ffff8d25c8021500 x1876699507456256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761178.364118:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507456256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761203 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761178.364122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761178.364124:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761178.364126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507456256:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761178.427527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761179.451504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761180.475547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761181.499498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761182.523546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761183.483492:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761183.483500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761183.483503:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761183.483511:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1161 00000100:00080000:3.0:1789761183.483520:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761183.483525:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761183.483557:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507456384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761183.483579:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761183.483581:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761183.483583:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761183.483586:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1161 00000100:00080000:3.0:1789761183.483589:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761183.483591:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761183.483599:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761183.483607:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507456512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761183.483616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761183.483620:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507456384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761183.483621:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761183.483623:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761183.483630:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1161 00000100:00080000:3.0:1789761183.483633:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761183.483636:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761183.483641:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761183.483649:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507456640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761183.483656:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761183.483659:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761183.483661:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761183.483663:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1161 00000100:00080000:3.0:1789761183.483665:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761183.483667:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761183.483671:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789761183.483692:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761183.483694:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507456768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761183.483704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761183.483707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761183.483709:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761183.483712:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1161 00000100:00080000:3.0:1789761183.483715:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761183.483718:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761183.483723:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761183.483733:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761183/real 1789761183] req@ffff8d25d1b0ad80 x1876699507456384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761183.483735:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507456896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761183.483748:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761183.483755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761183.483758:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761183.483763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1161 00000100:00080000:3.0:1789761183.483766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761183.483769:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761183.483772:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761183.483773:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507456384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761183.483781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507457024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761183.483785:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761183.483794:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761183.483800:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507456384:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761183.483822:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761183.483839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761183.483849:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761183/real 1789761183] req@ffff8d25d1b0b100 x1876699507456512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761183.483857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507456512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761183.483863:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761183.483865:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761183.483867:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761183.483871:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761183.483881:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761183.483887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761183/real 1789761183] req@ffff8d25d1b0aa00 x1876699507456640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761183.483900:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507456640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761183.483905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761183.483907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761183.483908:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761183.483912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761183.483922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761183.483928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761183/real 1789761183] req@ffff8d25d1b09180 x1876699507456768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761183.483936:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507456768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761183.483941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761183.483943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761183.483944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507456768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761183.483948:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507456896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761183.483958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761183.483963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761183/real 1789761183] req@ffff8d25d1b09c00 x1876699507456896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761183.483971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507456896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761183.483980:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761183.483982:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761183.483984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507456896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761183.483987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507457024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761183.483997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761183.484003:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761183/real 1789761183] req@ffff8d25d1b0bb80 x1876699507457024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761183.484010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507457024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761208 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761183.484015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761183.484017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761183.484018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507457024:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761183.547535:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761184.571559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761185.595503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761186.619494:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761187.643528:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761188.603478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761188.603486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761188.603490:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761188.603498:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1166 00000100:00080000:2.0:1789761188.603506:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761188.603512:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761188.603544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507457152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761188.603571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761188.603576:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761188.603578:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761188.603583:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1166 00000100:00080000:2.0:1789761188.603587:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761188.603593:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761188.603601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507457152:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761188.603603:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761188.603624:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507457280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761188.603638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761188.603653:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761188.603655:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761188.603659:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1166 00000100:00080000:2.0:1789761188.603663:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761188.603666:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761188.603671:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761188.603681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507457408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761188.603695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761188.603697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761188.603699:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761188.603702:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1166 00000100:00080000:2.0:1789761188.603705:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761188.603710:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761188.603712:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761188.603717:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761188.603725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507457536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761188.603732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761188.603735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761188.603737:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761188.603742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1166 00000100:00080000:2.0:1789761188.603747:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761188.603752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761188.603758:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761188.603759:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761188/real 1789761188] req@ffff8d25c8020000 x1876699507457152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761188.603773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507457664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761188.603787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761188.603788:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507457152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761188.603794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761188.603800:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761188.603810:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1166 00000100:00080000:0.0:1789761188.603811:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761188.603816:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761188.603820:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761188.603825:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761188.603827:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507457152:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761188.603835:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761188.603851:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507457792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761188.603856:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507457280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761188.603889:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761188.603904:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761188/real 1789761188] req@ffff8d25c8023100 x1876699507457280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761188.603916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507457280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761188.603925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761188.603928:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761188.603932:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507457280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761188.603938:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507457408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761188.603956:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761188.603966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761188/real 1789761188] req@ffff8d25c8022300 x1876699507457408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761188.603977:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507457408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761188.603985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761188.603988:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761188.603991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507457408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761188.603997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507457536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761188.604013:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761188.604022:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761188/real 1789761188] req@ffff8d25c8022d80 x1876699507457536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761188.604040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507457536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761188.604049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761188.604052:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761188.604056:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507457536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761188.604062:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507457664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761188.604080:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761188.604090:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761188/real 1789761188] req@ffff8d25c8020700 x1876699507457664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761188.604102:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507457664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761188.604110:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761188.604113:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761188.604123:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507457664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761188.604140:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507457792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761188.604159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761188.604170:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761188/real 1789761188] req@ffff8d25c8023b80 x1876699507457792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761188.604188:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507457792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761213 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761188.604197:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761188.604200:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761188.604203:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507457792:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761188.667536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761189.691513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761190.715495:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761191.739496:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761192.763502:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761193.723462:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761193.723470:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761193.723474:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761193.723483:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1171 00000100:00080000:2.0:1789761193.723493:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761193.723502:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761193.723536:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507457920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761193.723562:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761193.723565:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761193.723567:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761193.723571:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1171 00000100:00080000:2.0:1789761193.723576:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761193.723580:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761193.723589:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761193.723612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507457920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761193.723618:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507458048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761193.723636:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761193.723640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761193.723643:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761193.723649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1171 00000100:00080000:2.0:1789761193.723662:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761193.723667:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761193.723674:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761193.723676:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761193.723699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507458176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761193.723710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761193.723715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761193.723716:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761193.723721:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1171 00000100:00080000:2.0:1789761193.723725:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761193.723730:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761193.723734:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761193.723742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507458304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761193.723749:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761193.723752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761193.723753:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761193.723755:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1171 00000100:00080000:2.0:1789761193.723758:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761193.723761:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761193.723765:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761193.723771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507458432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761193.723779:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761193/real 1789761193] req@ffff8d25c8021180 x1876699507457920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761193.723780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761193.723788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761193.723791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761193.723799:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1171 00000100:00080000:2.0:1789761193.723802:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761193.723805:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761193.723808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761193.723812:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507457920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761193.723822:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507458560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761193.723823:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761193.723832:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761193.723839:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507457920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761193.723864:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761193.723977:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761193.723990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761193/real 1789761193] req@ffff8d25c8020380 x1876699507458048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761193.724066:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507458048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761193.724077:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761193.724081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761193.724085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761193.724092:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761193.724126:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761193.724136:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761193/real 1789761193] req@ffff8d25c8023800 x1876699507458176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761193.724145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507458176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761193.724150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761193.724153:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761193.724155:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761193.724159:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761193.724171:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761193.724182:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761193/real 1789761193] req@ffff8d25c8022680 x1876699507458304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761193.724191:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507458304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761193.724217:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761193.724220:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761193.724222:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761193.724225:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507458432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761193.724237:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761193.724244:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761193/real 1789761193] req@ffff8d25c8022a00 x1876699507458432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761193.724251:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507458432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761193.724257:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761193.724259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761193.724260:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507458432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761193.724264:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507458560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761193.724298:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761193.724309:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761193/real 1789761193] req@ffff8d25c8020a80 x1876699507458560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761193.724322:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507458560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761218 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761193.724347:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761193.724351:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761193.724355:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507458560:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761193.787584:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761194.811499:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761195.835578:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761196.859485:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761197.883532:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761198.843477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761198.843487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761198.843491:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761198.843500:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1176 00000100:00080000:2.0:1789761198.843507:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761198.843513:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761198.843546:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507458688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761198.843586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761198.843591:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761198.843593:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761198.843598:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1176 00000100:00080000:2.0:1789761198.843603:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761198.843609:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761198.843620:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761198.843627:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507458688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761198.843640:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507458816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761198.843654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761198.843658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761198.843660:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761198.843664:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1176 00000100:00080000:2.0:1789761198.843668:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761198.843671:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761198.843676:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761198.843687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507458944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761198.843695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761198.843698:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761198.843699:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761198.843702:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1176 00000100:00080000:2.0:1789761198.843706:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761198.843714:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761198.843719:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761198.843727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507459072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761198.843734:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761198.843745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761198.843750:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761198.843753:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761198.843760:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1176 00000100:00080000:2.0:1789761198.843765:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761198.843768:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458944:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761198.843774:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761198.843782:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761198.843792:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761198.843799:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507459200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761198.843809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761198.843812:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761198.843814:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761198.843816:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507459200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761198.843820:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1176 00000100:00080000:2.0:1789761198.843826:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761198.843830:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761198.843834:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761198.843845:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507459328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761198.843883:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507459328:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761198.907502:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761199.931531:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761200.955512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761201.915630:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761201.915671:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761201.915683:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761201.915716:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761201.915726:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761201.915737:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761201.915750:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761201.915761:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761201.915762:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761198/real 1789761201] req@ffff8d25c8023800 x1876699507459072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761201.915791:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507459072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761201.915804:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761201.915810:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761201.915821:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761201.915830:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761198/real 1789761201] req@ffff8d25c8022a00 x1876699507458816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761201.915846:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507458816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761201.915876:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761201.915881:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761201.915887:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761201.915894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761198/real 1789761201] req@ffff8d25c8022680 x1876699507458944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761201.915906:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507458944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761201.915915:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761201.915919:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761201.915923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507458944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761201.915931:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761198/real 1789761201] req@ffff8d25c8020a80 x1876699507458688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761201.915942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507458688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761201.915950:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761201.915954:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761201.915957:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507458688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761201.915985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761198/real 1789761201] req@ffff8d25c8021180 x1876699507459328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761201.915998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507459328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761201.916005:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761201.916009:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761201.916012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507459328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761201.916019:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761198/real 1789761201] req@ffff8d25c8020380 x1876699507459200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761201.916031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507459200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761223 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761201.916038:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761201.916041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761201.916045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507459200:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761201.979515:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761203.003490:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761203.963452:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761203.963461:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761203.963464:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761203.963474:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1181 00000100:00080000:2.0:1789761203.963483:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761203.963489:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761203.963517:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507459456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761203.963555:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761203.963558:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761203.963560:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761203.963563:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1181 00000100:00080000:2.0:1789761203.963566:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761203.963569:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761203.963577:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761203.963588:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507459584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761203.963599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761203.963603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761203.963605:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761203.963611:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1181 00000100:00100000:3.0:1789761203.963613:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507459456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761203.963615:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761203.963626:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761203.963632:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761203.963647:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507459712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761203.963660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761203.963664:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761203.963666:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761203.963670:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1181 00000100:00080000:2.0:1789761203.963675:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761203.963679:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761203.963685:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761203.963690:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761203.963702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507459840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761203.963715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761203.963718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761203.963721:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761203.963725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1181 00000100:00080000:2.0:1789761203.963730:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761203.963731:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761203/real 1789761203] req@ffff8d25c8023b80 x1876699507459456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761203.963739:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761203.963750:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761203.963755:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507459456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761203.963769:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761203.963774:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761203.963778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507459968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761203.963783:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507459456:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761203.963799:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761203.963801:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761203.963809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761203.963811:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761203.963816:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1181 00000100:00080000:2.0:1789761203.963820:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761203.963824:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761203.963831:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761203.963835:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761203.963850:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507460096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761203.963856:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761203/real 1789761203] req@ffff8d25c8020700 x1876699507459584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761203.963883:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507459584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761203.963893:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761203.963897:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761203.963901:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761203.963926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761203.963950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761203.963962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761203/real 1789761203] req@ffff8d25c8022d80 x1876699507459712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761203.963976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507459712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761203.963986:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761203.963989:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761203.963992:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761203.964000:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761203.964021:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761203.964033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761203/real 1789761203] req@ffff8d25c8022300 x1876699507459840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761203.964053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507459840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761203.964061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761203.964064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761203.964067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507459840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761203.964073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507459968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761203.964093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761203.964106:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761203/real 1789761203] req@ffff8d25c8023100 x1876699507459968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761203.964128:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507459968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761203.964137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761203.964141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761203.964145:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507459968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761203.964151:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507460096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761203.964257:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761203.964271:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761203/real 1789761203] req@ffff8d25c8020000 x1876699507460096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761203.964283:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507460096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761228 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761203.964297:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761203.964300:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761203.964304:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507460096:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761204.027549:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761205.051555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761206.075535:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761207.099543:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761208.123511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761209.083506:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761209.083516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761209.083520:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761209.083532:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1186 00000100:00080000:0.0:1789761209.083544:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761209.083553:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761209.083604:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507460224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761209.083631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761209.083635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761209.083636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761209.083640:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1186 00000100:00080000:0.0:1789761209.083643:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761209.083647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761209.083657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761209.083669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507460352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761209.083679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761209.083681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761209.083684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761209.083691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1186 00000100:00100000:3.0:1789761209.083691:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507460224:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761209.083700:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761209.083711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761209.083714:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761209.083724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507460480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761209.083732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761209.083735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761209.083736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761209.083738:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1186 00000100:00080000:0.0:1789761209.083741:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761209.083744:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761209.083747:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761209.083756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507460608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761209.083766:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761209.083767:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761209.083775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761209.083777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761209.083788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1186 00000100:00080000:0.0:1789761209.083791:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761209.083795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761209.083800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761209.083817:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffff8d25d25b8a80 x1876699507460224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761209.083818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507460736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761209.083902:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761209.083909:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507460224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761209.083912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761209.083918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761209.083937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1186 00000100:00080000:0.0:1789761209.083942:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761209.083943:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761209.083957:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761209.083958:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761209.083973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761209.083974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507460224:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761209.083989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507460864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761209.084009:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507460352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761209.084043:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761209.084059:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffff8d25d25ba300 x1876699507460352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761209.084071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507460352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761209.084078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761209.084081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761209.084083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507460352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761209.084088:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507460480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761209.084105:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761209.084112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffff8d25d25b8000 x1876699507460480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761209.084120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507460480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761209.084125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761209.084127:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761209.084129:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507460480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761209.084133:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507460608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761209.084144:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761209.084151:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffff8d25d25bbb80 x1876699507460608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761209.084158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507460608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761209.084164:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761209.084165:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761209.084167:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507460608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761209.084171:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507460736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761209.084181:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761209.084188:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffff8d25d25ba680 x1876699507460736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761209.084195:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507460736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761209.084201:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761209.084202:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761209.084204:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507460736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761209.084208:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507460864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761209.084226:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761209.084234:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761209/real 1789761209] req@ffff8d25d25b9180 x1876699507460864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761209.084242:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507460864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761234 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761209.084248:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761209.084250:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761209.084252:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507460864:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761209.147565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761210.171546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761211.195501:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761212.219490:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761213.243532:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761214.203490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761214.203500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761214.203505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761214.203516:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1191 00000100:00080000:2.0:1789761214.203526:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761214.203535:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761214.203571:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507460992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761214.203592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761214.203595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761214.203596:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761214.203600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1191 00000100:00080000:2.0:1789761214.203604:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761214.203609:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761214.203618:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761214.203621:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507460992:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761214.203630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507461120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761214.203639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761214.203642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761214.203643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761214.203645:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1191 00000100:00080000:2.0:1789761214.203648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761214.203651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761214.203654:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761214.203662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507461248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761214.203668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761214.203670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761214.203671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761214.203674:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1191 00000800:00000100:3.0:1789761214.203678:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761214.203679:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761214.203685:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761214.203690:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761214.203703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507461376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761214.203710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761214.203712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761214.203714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761214.203717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1191 00000100:00080000:2.0:1789761214.203720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761214.203725:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761214.203727:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffff8d25c8021500 x1876699507460992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761214.203729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761214.203742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507461504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761214.203751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761214.203754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761214.203755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761214.203760:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507460992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761214.203761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1191 00000100:00080000:2.0:1789761214.203766:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761214.203771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761214.203777:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761214.203788:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761214.203790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507461632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761214.203800:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761214.203807:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507460992:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761214.203831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761214.203851:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761214.203861:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffff8d25c8020e00 x1876699507461120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761214.203872:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507461120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761214.203881:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761214.203885:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761214.203889:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761214.203895:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761214.203918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761214.203931:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffff8d25c8021c00 x1876699507461248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761214.203945:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507461248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761214.203971:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761214.203975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761214.203978:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761214.203985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761214.204004:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761214.204015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffff8d25c8021880 x1876699507461376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761214.204027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507461376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761214.204036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761214.204038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761214.204040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761214.204045:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507461504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761214.204058:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761214.204066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffff8d25c8023480 x1876699507461504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761214.204073:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507461504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761214.204078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761214.204080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761214.204083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507461504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761214.204087:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507461632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761214.204097:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761214.204103:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761214/real 1789761214] req@ffff8d25c8020000 x1876699507461632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761214.204110:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507461632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761239 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761214.204115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761214.204117:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761214.204119:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507461632:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761214.267542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761215.291595:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761216.315542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761217.339618:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761218.363514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761219.323542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761219.323560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761219.323566:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761219.323580:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1197 00000100:00080000:2.0:1789761219.323591:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761219.323599:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761219.323646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507461760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761219.323671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761219.323674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761219.323676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761219.323679:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1197 00000100:00080000:2.0:1789761219.323682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761219.323685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761219.323694:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761219.323704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507461888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761219.323711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761219.323714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761219.323715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761219.323720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1197 00000100:00100000:3.0:1789761219.323722:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507461760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761219.323723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761219.323727:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761219.323734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761219.323744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507462016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761219.323750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761219.323753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761219.323754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761219.323758:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1197 00000100:00080000:2.0:1789761219.323760:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761219.323762:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761219.323766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761219.323775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507462144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761219.323782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761219.323784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761219.323786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761219.323788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1197 00000100:00080000:2.0:1789761219.323798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761219.323801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761219.323804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761219.323811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507462272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761219.323820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761219.323820:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761219.323826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761219.323827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761219.323830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1197 00000100:00080000:2.0:1789761219.323834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761219.323836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761219.323839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761219.323846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507462400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761219.323881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761219] req@ffff8d25c8023100 x1876699507461760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761219.323900:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507461760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761219.323910:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761219.323913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761219.323916:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507461760:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761219.323945:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761219.323974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761219.323986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761219] req@ffff8d25c8022300 x1876699507461888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761219.324000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507461888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761219.324009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761219.324013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761219.324016:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507461888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761219.324023:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761219.324047:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761219.324061:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761219] req@ffff8d25c8022d80 x1876699507462016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761219.324073:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507462016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761219.324079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761219.324081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761219.324083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761219.324087:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761219.324103:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761219.324111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761219] req@ffff8d25c8020700 x1876699507462144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761219.324123:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507462144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761219.324128:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761219.324130:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761219.324132:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761219.324135:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507462272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761219.324147:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761219.324153:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761219] req@ffff8d25c8023b80 x1876699507462272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761219.324161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507462272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761219.324166:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761219.324167:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761219.324169:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507462272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761219.324172:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507462400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761219.324182:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761219.324188:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761219/real 1789761219] req@ffff8d25c8020380 x1876699507462400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761219.324195:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507462400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761244 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761219.324204:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761219.324206:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761219.324208:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507462400:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761219.387546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761220.411536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761221.435597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761222.459584:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761223.483554:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761224.443569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761224.443580:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761224.443585:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761224.443596:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1202 00000100:00080000:1.0:1789761224.443608:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761224.443614:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761224.443646:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507462528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761224.443678:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761224.443683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761224.443685:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761224.443691:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1202 00000100:00080000:1.0:1789761224.443695:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761224.443700:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761224.443711:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761224.443727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507462656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761224.443738:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507462528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761224.443742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761224.443749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761224.443751:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761224.443756:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1202 00000100:00080000:1.0:1789761224.443761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761224.443774:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761224.443781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761224.443795:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507462784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761224.443807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761224.443809:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761224.443814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761224.443818:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761224.443823:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1202 00000100:00080000:1.0:1789761224.443828:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761224.443833:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761224.443839:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761224.443852:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507462912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761224.443863:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761224.443867:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761224/real 1789761224] req@ffff8d25c8168000 x1876699507462528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761224.443868:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761224.443871:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761224.443881:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1202 00000100:00080000:1.0:1789761224.443891:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761224.443896:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761224.443903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507462528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761224.443904:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761224.443920:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789761224.443922:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507463040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761224.443930:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761224.443940:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507462528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761224.443941:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761224.443949:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761224.443952:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761224.443958:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1202 00000100:00080000:1.0:1789761224.443962:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761224.443967:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761224.443973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462656:192.168.203.141@tcp:38:kworker.0 00000008:00100000:1.0:1789761224.443974:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761224.443992:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507463168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761224.444010:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761224.444027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761224/real 1789761224] req@ffff8d25c8168380 x1876699507462656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761224.444040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507462656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761224.444047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761224.444050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761224.444052:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761224.444057:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761224.444080:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761224.444088:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761224/real 1789761224] req@ffff8d25c816b480 x1876699507462784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761224.444097:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507462784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761224.444102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761224.444104:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761224.444106:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761224.444113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761224.444125:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761224.444132:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761224/real 1789761224] req@ffff8d25c816a300 x1876699507462912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761224.444144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507462912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761224.444152:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761224.444156:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761224.444159:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507462912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761224.444166:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761224.444188:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761224.444198:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761224/real 1789761224] req@ffff8d25c816ad80 x1876699507463040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761224.444207:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507463040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761224.444213:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761224.444215:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761224.444217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761224.444230:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761224.444243:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761224.444249:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761224/real 1789761224] req@ffff8d25c8168a80 x1876699507463168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761224.444257:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507463168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761249 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761224.444262:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761224.444264:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761224.444266:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463168:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761224.507606:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761225.531590:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761226.555576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761227.579557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761228.603489:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761229.563498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761229.563506:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761229.563511:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761229.563520:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1207 00000100:00080000:3.0:1789761229.563529:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761229.563535:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761229.563564:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507463296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761229.563589:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761229.563592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761229.563594:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761229.563598:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1207 00000100:00080000:3.0:1789761229.563600:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761229.563603:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761229.563611:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761229.563622:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507463424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761229.563632:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507463296:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761229.563633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761229.563642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761229.563645:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761229.563649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1207 00000100:00080000:3.0:1789761229.563652:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761229.563656:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761229.563660:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761229.563671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507463552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761229.563677:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761229.563680:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761229.563682:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761229.563685:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1207 00000100:00080000:3.0:1789761229.563688:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761229.563691:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761229.563695:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761229.563696:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761229.563709:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507463680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761229.563736:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761229/real 1789761229] req@ffff8d25d1b08700 x1876699507463296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761229.563750:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507463296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761229.563758:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761229.563761:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761229.563765:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507463296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761229.563788:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507463424:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761229.563790:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761229.563796:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761229.563798:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761229.563802:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1207 00000100:00080000:3.0:1789761229.563805:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761229.563808:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761229.563811:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761229.563822:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761229/real 1789761229] req@ffff8d25d1b0b800 x1876699507463424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761229.563830:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507463424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761229.563838:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761229.563842:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761229.563845:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507463424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761229.563860:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507463552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761229.563886:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761229.563901:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761229/real 1789761229] req@ffff8d25d1b08e00 x1876699507463552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761229.563909:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761229.563927:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507463808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761229.563934:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507463552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761229.563941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761229.563943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761229.563945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507463552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761229.563966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507463680:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789761229.563970:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761229.563975:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761229.563977:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761229.563982:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1207 00000100:00080000:3.0:1789761229.563986:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761229.563990:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761229.563996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761229.563998:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761229.564011:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507463936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761229.564017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761229/real 1789761229] req@ffff8d25d1b09880 x1876699507463680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761229.564035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507463680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761229.564045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761229.564047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761229.564050:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507463680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761229.564060:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761229.564076:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761229.564084:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761229/real 1789761229] req@ffff8d25d1b0a680 x1876699507463808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761229.564092:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507463808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761229.564101:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761229.564103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761229.564105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761229.564109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761229.564119:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761229.564125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761229/real 1789761229] req@ffff8d25d1b08e00 x1876699507463936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761229.564132:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507463936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761254 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761229.564137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761229.564139:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761229.564141:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507463936:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761229.627522:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761230.651534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761231.675552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761232.699527:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761233.723506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761234.683457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761234.683468:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761234.683472:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761234.683482:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1212 00000100:00080000:0.0:1789761234.683491:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761234.683500:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761234.683536:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507464064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761234.683573:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761234.683577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761234.683579:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761234.683585:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1212 00000100:00080000:0.0:1789761234.683590:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761234.683595:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761234.683603:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507464064:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761234.683607:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761234.683626:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507464192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761234.683639:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761234.683643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761234.683645:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761234.683649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1212 00000100:00080000:0.0:1789761234.683654:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761234.683658:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761234.683665:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761234.683668:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761234.683683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507464320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761234.683694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761234.683697:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761234.683699:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761234.683705:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1212 00000100:00100000:3.0:1789761234.683706:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761234/real 1789761234] req@ffff8d25d25b8380 x1876699507464064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761234.683713:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761234.683722:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761234.683727:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507464064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761234.683730:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:3.0:1789761234.683741:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761234.683744:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761234.683752:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507464064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761234.683753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507464448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761234.683768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761234.683772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761234.683776:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761234.683776:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464192:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761234.683787:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1212 00000100:00080000:0.0:1789761234.683793:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761234.683799:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761234.683804:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789761234.683809:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761234.683816:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761234/real 1789761234] req@ffff8d25d25bb800 x1876699507464192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761234.683826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507464576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761234.683830:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507464192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761234.683855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761234.683864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761234.683864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761234.683870:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761234.683871:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761234.683879:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1212 00000100:00100000:3.0:1789761234.683880:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464192:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761234.683889:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761234.683890:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464320:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761234.683899:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761234.683908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761234.683912:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761234.683923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761234/real 1789761234] req@ffff8d25d25b9f80 x1876699507464320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761234.683938:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507464704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761234.683938:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507464320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761234.683970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761234.683975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761234.683982:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761234.683991:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761234.684006:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761234.684015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761234/real 1789761234] req@ffff8d25d25bad80 x1876699507464448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761234.684022:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507464448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761234.684027:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761234.684030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761234.684033:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761234.684049:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507464576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761234.684070:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761234.684084:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761234/real 1789761234] req@ffff8d25d25b8700 x1876699507464576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761234.684097:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507464576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761234.684107:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761234.684110:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761234.684114:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507464576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761234.684120:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507464704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761234.684144:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761234.684153:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761234/real 1789761234] req@ffff8d25d25bb100 x1876699507464704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761234.684161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507464704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761259 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761234.684166:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761234.684169:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761234.684171:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507464704:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761234.747493:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761235.771498:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761236.795509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761237.819497:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761238.843538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761239.803469:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761239.803476:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761239.803480:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761239.803491:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1217 00000100:00080000:2.0:1789761239.803501:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761239.803509:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761239.803544:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507464832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761239.803566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761239.803569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761239.803570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761239.803574:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1217 00000100:00080000:2.0:1789761239.803578:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761239.803582:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761239.803590:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761239.803601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507464960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761239.803607:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507464832:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761239.803612:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761239.803616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761239.803618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761239.803621:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1217 00000100:00080000:2.0:1789761239.803623:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761239.803625:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761239.803628:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761239.803637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507465088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761239.803643:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761239.803646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761239.803647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761239.803657:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1217 00000100:00080000:2.0:1789761239.803660:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761239.803663:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761239.803667:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761239.803669:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761239.803680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507465216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761239.803687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761239.803690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761239.803691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761239.803694:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1217 00000100:00080000:2.0:1789761239.803697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761239.803700:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761239.803705:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761239/real 1789761239] req@ffff8d25c8021180 x1876699507464832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761239.803706:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761239.803719:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507465344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761239.803741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761239.803744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761239.803748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761239.803749:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507464832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761239.803753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1217 00000100:00080000:2.0:1789761239.803758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761239.803763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761239.803769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761239.803775:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761239.803780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507465472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761239.803781:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761239.803789:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507464832:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761239.803815:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761239.803835:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761239.803844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761239/real 1789761239] req@ffff8d25c8020a80 x1876699507464960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761239.803853:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507464960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761239.803859:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761239.803861:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761239.803863:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507464960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761239.803867:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761239.803878:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761239.803885:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761239/real 1789761239] req@ffff8d25c8022680 x1876699507465088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761239.803893:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507465088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761239.803909:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761239.803912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761239.803916:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761239.803923:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761239.803943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761239.803954:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761239/real 1789761239] req@ffff8d25c8022a00 x1876699507465216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761239.803962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507465216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761239.803968:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761239.803971:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761239.803973:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761239.803976:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507465344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761239.803990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761239.803997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761239/real 1789761239] req@ffff8d25c8023800 x1876699507465344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761239.804004:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507465344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761239.804009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761239.804011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761239.804014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507465344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761239.804020:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507465472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761239.804039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761239.804050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761239/real 1789761239] req@ffff8d25c8020380 x1876699507465472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761239.804063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507465472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761264 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761239.804071:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761239.804074:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761239.804076:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507465472:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761239.867488:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761240.891523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761241.915556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761242.939576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761243.963544:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761244.923468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761244.923479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761244.923484:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761244.923495:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1222 00000100:00080000:3.0:1789761244.923504:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761244.923512:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761244.923551:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507465600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761244.923579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761244.923582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761244.923584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761244.923588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1222 00000100:00080000:3.0:1789761244.923593:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761244.923597:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761244.923605:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761244.923612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507465600:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761244.923619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507465728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761244.923628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761244.923630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761244.923631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761244.923634:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1222 00000100:00080000:3.0:1789761244.923636:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761244.923639:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761244.923642:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761244.923650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507465856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761244.923657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761244.923660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761244.923661:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761244.923663:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1222 00000100:00080000:3.0:1789761244.923665:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761244.923671:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761244.923672:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761244.923680:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761244.923703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507465984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761244.923712:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761244/real 1789761244] req@ffff8d25d1b0b800 x1876699507465600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761244.923749:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761244.923750:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507465600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761244.923760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761244.923766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761244.923767:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761244.923774:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761244.923777:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1222 00000100:00100000:0.0:1789761244.923780:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507465600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761244.923783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761244.923790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761244.923795:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761244.923804:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761244.923808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507466112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761244.923819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761244.923822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761244.923824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761244.923827:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761244.923831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1222 00000100:00080000:3.0:1789761244.923836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761244.923839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761244.923843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761244.923844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761244/real 1789761244] req@ffff8d25d1b08700 x1876699507465728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761244.923855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507466240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761244.923858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507465728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761244.923880:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761244.923884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761244.923887:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761244.923895:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761244.923908:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761244.923915:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761244/real 1789761244] req@ffff8d25d1b0bb80 x1876699507465856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761244.923923:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507465856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761244.923928:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761244.923929:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761244.923931:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761244.923935:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761244.923945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761244.923951:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761244/real 1789761244] req@ffff8d25d1b09c00 x1876699507465984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761244.923958:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507465984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761244.923963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761244.923965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761244.923966:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507465984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761244.923979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507466112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761244.923990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761244.923996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761244/real 1789761244] req@ffff8d25d1b09180 x1876699507466112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761244.924004:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507466112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761244.924009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761244.924010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761244.924012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507466112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761244.924016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507466240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761244.924025:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761244.924031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761244/real 1789761244] req@ffff8d25d1b0aa00 x1876699507466240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761244.924038:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507466240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761269 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761244.924043:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761244.924045:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761244.924047:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507466240:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761244.987546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761246.011546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761247.035530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761248.059539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761249.083560:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761250.043523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761250.043532:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761250.043537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761250.043549:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1227 00000100:00080000:2.0:1789761250.043558:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761250.043566:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761250.043607:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507466368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761250.043631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761250.043635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761250.043636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761250.043640:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1227 00000100:00080000:2.0:1789761250.043643:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761250.043646:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761250.043657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761250.043675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507466496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761250.043685:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507466368:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761250.043691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761250.043696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761250.043698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761250.043704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1227 00000100:00080000:2.0:1789761250.043708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761250.043711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761250.043715:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761250.043727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507466624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761250.043735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761250.043738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761250.043740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761250.043744:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1227 00000100:00080000:2.0:1789761250.043748:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761250.043753:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761250.043760:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761250.043761:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761250.043780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507466752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761250.043792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761250.043796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761250.043798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761250.043803:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1227 00000100:00080000:2.0:1789761250.043807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761250.043812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761250.043817:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffff8d25c8023b80 x1876699507466368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761250.043820:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761250.043835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507466880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761250.043852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761250.043856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761250.043860:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507466368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761250.043861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761250.043875:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761250.043878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1227 00000100:00080000:0.0:1789761250.043882:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761250.043884:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761250.043888:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507466368:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761250.043892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761250.043898:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761250.043907:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507467008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761250.043919:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507466496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761250.043944:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761250.043956:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffff8d25c8020700 x1876699507466496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761250.043964:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507466496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761250.043970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761250.043973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761250.043975:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507466496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761250.043979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507466624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761250.043990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761250.043997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffff8d25c8022d80 x1876699507466624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761250.044005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507466624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761250.044010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761250.044012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761250.044014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507466624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761250.044018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507466752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761250.044029:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761250.044035:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffff8d25c8022300 x1876699507466752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761250.044043:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507466752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761250.044049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761250.044050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761250.044052:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507466752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761250.044055:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507466880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761250.044065:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761250.044071:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffff8d25c8023100 x1876699507466880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761250.044078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507466880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761250.044082:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761250.044084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761250.044087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507466880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761250.044099:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507467008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761250.044109:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761250.044116:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761250/real 1789761250] req@ffff8d25c8020000 x1876699507467008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761250.044124:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507467008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761275 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761250.044129:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761250.044131:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761250.044133:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507467008:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761250.107560:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761251.131582:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761252.155613:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761253.179570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761254.203550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761255.163531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761255.163538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761255.163542:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761255.163551:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1232 00000100:00080000:3.0:1789761255.163560:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761255.163565:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761255.163595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507467136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761255.163620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761255.163624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761255.163626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761255.163632:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1232 00000100:00080000:3.0:1789761255.163635:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761255.163640:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761255.163650:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761255.163665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507467264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761255.163674:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507467136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761255.163680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761255.163684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761255.163686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761255.163689:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1232 00000100:00080000:3.0:1789761255.163692:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761255.163695:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761255.163699:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761255.163710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507467392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761255.163726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761255.163729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761255.163730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761255.163733:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1232 00000100:00080000:3.0:1789761255.163736:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761255.163741:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761255.163743:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761255.163749:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761255.163757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507467520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761255.163765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761255.163768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761255.163769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761255.163771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1232 00000100:00080000:3.0:1789761255.163773:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761255.163777:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761255.163787:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761255.163792:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffff8d25d1b0b100 x1876699507467136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761255.163804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507467648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761255.163817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761255.163821:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761255.163826:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507467136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761255.163826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761255.163836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1232 00000100:00080000:3.0:1789761255.163844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761255.163848:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761255.163851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761255.163857:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789761255.163861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761255.163865:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507467136:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761255.163878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507467776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761255.163899:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507467264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761255.163926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761255.163940:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffff8d25d1b0ad80 x1876699507467264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761255.163954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507467264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761255.163971:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761255.163975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761255.163979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507467264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761255.163985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507467392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761255.164005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761255.164016:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffff8d25d1b08a80 x1876699507467392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761255.164029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507467392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761255.164038:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761255.164042:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761255.164045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507467392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761255.164052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507467520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761255.164070:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761255.164080:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffff8d25d1b08380 x1876699507467520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761255.164093:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507467520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761255.164102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761255.164105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761255.164110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507467520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761255.164116:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507467648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761255.164133:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761255.164159:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffff8d25d1b0a300 x1876699507467648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761255.164174:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507467648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761255.164183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761255.164186:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761255.164190:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507467648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761255.164196:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507467776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761255.164215:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761255.164227:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761255/real 1789761255] req@ffff8d25d1b0b480 x1876699507467776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761255.164240:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507467776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761280 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761255.164248:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761255.164251:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761255.164255:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507467776:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761255.227518:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761256.251552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761257.275511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761258.299534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761259.323496:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761260.283487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761260.283498:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761260.283503:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761260.283514:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1237 00000100:00080000:3.0:1789761260.283524:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761260.283534:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761260.283585:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507467904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761260.283617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761260.283620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761260.283621:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761260.283625:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1237 00000100:00080000:3.0:1789761260.283628:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761260.283632:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761260.283641:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761260.283653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507468032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761260.283661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761260.283664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761260.283665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761260.283668:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1237 00000100:00080000:3.0:1789761260.283671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761260.283676:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761260.283678:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507467904:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789761260.283680:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761260.283692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507468160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761260.283699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761260.283702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761260.283703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761260.283706:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1237 00000100:00080000:3.0:1789761260.283708:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761260.283710:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761260.283713:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761260.283721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507468288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761260.283728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761260.283730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761260.283731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761260.283734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1237 00000100:00080000:3.0:1789761260.283736:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761260.283739:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761260.283742:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761260.283748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507468416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761260.283754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761260.283757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761260.283759:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761260.283761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1237 00000100:00080000:3.0:1789761260.283763:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761260.283765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761260.283767:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761260.283776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507468544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761260.283778:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761260.283838:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffff8d25d1b08000 x1876699507467904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761260.283859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507467904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761260.283879:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761260.283884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761260.283889:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507467904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761260.283897:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761260.283921:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761260.283944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffff8d25d1b09500 x1876699507468032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761260.283957:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507468032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761260.283965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761260.283968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761260.283972:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761260.284004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761260.284023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761260.284034:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffff8d25d1b0b480 x1876699507468160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761260.284053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507468160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761260.284060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761260.284063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761260.284067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761260.284073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761260.284093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761260.284103:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffff8d25d1b0a300 x1876699507468288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761260.284130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507468288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761260.284138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761260.284141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761260.284145:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761260.284152:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507468416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761260.284181:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761260.284193:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffff8d25d1b08380 x1876699507468416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761260.284206:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507468416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761260.284220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761260.284223:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761260.284226:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507468416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761260.284241:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507468544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761260.284261:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761260.284273:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761260/real 1789761260] req@ffff8d25d1b08a80 x1876699507468544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761260.284283:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507468544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761285 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761260.284292:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761260.284296:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761260.284300:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507468544:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761260.347550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761261.371580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761262.395560:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761263.419596:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761264.443525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761265.403526:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761265.403540:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761265.403546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761265.403560:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1243 00000100:00080000:3.0:1789761265.403574:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761265.403584:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761265.403635:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507468672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761265.403671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761265.403675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761265.403678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761265.403683:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1243 00000100:00080000:3.0:1789761265.403687:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761265.403692:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761265.403703:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761265.403718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507468800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761265.403729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761265.403733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761265.403735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761265.403740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1243 00000100:00080000:3.0:1789761265.403745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761265.403746:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507468672:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761265.403753:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761265.403759:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761265.403773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507468928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761265.403785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761265.403788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761265.403790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761265.403794:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1243 00000100:00080000:3.0:1789761265.403799:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761265.403802:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761265.403807:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761265.403820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507469056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761265.403831:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761265.403835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761265.403837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761265.403841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1243 00000100:00080000:3.0:1789761265.403844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761265.403848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761265.403852:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761265.403865:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507469184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761265.403868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761265.403884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761265.403888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761265.403891:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761265.403896:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1243 00000100:00080000:3.0:1789761265.403901:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761265.403906:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761265.403911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761265.403915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761265.403927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507469312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761265.403946:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761265.403968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507469184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761265.403995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507469312:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761265.467583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761266.491585:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761267.515518:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761268.475574:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761268.475614:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761268.475630:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761268.475670:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761268.475684:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761268.475696:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761268.475707:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761265/real 1789761268] req@ffff8d25d1b0aa00 x1876699507468928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1789761268.475707:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761268.475721:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761268.475745:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507468928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761268.475759:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761268.475764:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761268.475768:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761268.475776:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761265/real 1789761268] req@ffff8d25d1b09180 x1876699507469056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761268.475789:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507469056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761268.475797:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761268.475801:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761268.475805:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761268.475812:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761265/real 1789761268] req@ffff8d25d1b09c00 x1876699507469184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761268.475824:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507469184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761268.475832:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761268.475836:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761268.475840:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507469184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761268.475846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761265/real 1789761268] req@ffff8d25d1b0ad80 x1876699507468672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761268.475859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507468672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761268.475867:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761268.475870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761268.475874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507468672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761268.475881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761265/real 1789761268] req@ffff8d25d1b0b100 x1876699507468800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761268.475893:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507468800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761268.475901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761268.475905:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761268.475908:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507468800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761268.475963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761265/real 1789761268] req@ffff8d25d1b0bb80 x1876699507469312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761268.475984:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507469312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761290 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761268.475993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761268.475998:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761268.476003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507469312:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761268.539521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761269.563539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761270.523473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761270.523481:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761270.523486:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761270.523495:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1248 00000100:00080000:3.0:1789761270.523502:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761270.523507:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761270.523534:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507469440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761270.523560:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761270.523565:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761270.523567:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761270.523572:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1248 00000100:00080000:3.0:1789761270.523576:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761270.523581:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761270.523591:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761270.523604:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507469568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761270.523614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761270.523617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761270.523618:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761270.523623:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1248 00000100:00100000:0.0:1789761270.523624:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507469440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761270.523626:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761270.523632:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761270.523637:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761270.523654:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507469696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761270.523672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761270.523675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761270.523678:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761270.523682:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1248 00000100:00080000:3.0:1789761270.523687:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761270.523694:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761270.523695:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761270.523705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761270.523720:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507469824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761270.523731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761270.523735:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761270.523737:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761270.523743:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761270/real 1789761270] req@ffff8d25d1b08700 x1876699507469440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761270.523745:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1248 00000100:00080000:3.0:1789761270.523753:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761270.523762:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761270.523768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761270.523774:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507469440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761270.523781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507469952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761270.523797:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761270.523798:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761270.523806:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761270.523808:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761270.523810:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761270.523814:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1248 00000100:00100000:0.0:1789761270.523820:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507469440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761270.523821:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761270.523831:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761270.523837:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761270.523849:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507470080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761270.523855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761270.523891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761270.523906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761270/real 1789761270] req@ffff8d25d1b0b800 x1876699507469568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761270.523920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507469568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761270.523930:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761270.523934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761270.523938:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761270.523945:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761270.523966:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761270.523978:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761270/real 1789761270] req@ffff8d25d1b08e00 x1876699507469696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761270.523990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507469696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761270.524000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761270.524004:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761270.524008:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761270.524014:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761270.524052:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761270.524065:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761270/real 1789761270] req@ffff8d25d1b0a680 x1876699507469824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761270.524078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507469824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761270.524088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761270.524091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761270.524095:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507469824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761270.524102:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507469952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761270.524123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761270.524136:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761270/real 1789761270] req@ffff8d25d1b09880 x1876699507469952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761270.524157:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507469952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761270.524166:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761270.524169:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761270.524172:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507469952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761270.524212:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507470080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761270.524235:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761270.524250:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761270/real 1789761270] req@ffff8d25d1b0bb80 x1876699507470080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761270.524262:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507470080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761295 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761270.524271:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761270.524274:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761270.524278:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507470080:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761270.587549:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761271.611487:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761272.635514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761273.659607:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761274.683499:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761275.643498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761275.643514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761275.643518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761275.643530:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1253 00000100:00080000:2.0:1789761275.643540:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761275.643547:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761275.643604:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507470208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761275.643631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761275.643635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761275.643637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761275.643643:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1253 00000100:00080000:2.0:1789761275.643648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761275.643653:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761275.643661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507470208:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761275.643663:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761275.643683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507470336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761275.643696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761275.643700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761275.643703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761275.643716:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761275.643717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1253 00000100:00080000:2.0:1789761275.643725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761275.643731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761275.643738:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761275.643753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507470464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761275.643755:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761275/real 1789761275] req@ffff8d25c8023480 x1876699507470208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761275.643786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761275.643789:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507470208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761275.643793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761275.643798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761275.643803:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761275.643807:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1253 00000100:00080000:0.0:1789761275.643810:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761275.643816:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507470208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761275.643817:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761275.643826:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761275.643832:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761275.643837:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507470336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761275.643848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507470592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761275.643857:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761275.643869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761275.643871:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761275/real 1789761275] req@ffff8d25c8021880 x1876699507470336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761275.643878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761275.643883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761275.643886:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507470336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761275.643892:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1253 00000100:00080000:0.0:1789761275.643895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761275.643898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761275.643900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761275.643905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507470336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761275.643909:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761275.643913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507470464:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789761275.643919:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761275.643930:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761275.643939:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507470720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761275.643940:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761275/real 1789761275] req@ffff8d25c8021c00 x1876699507470464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761275.643970:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507470464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761275.643973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761275.643978:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761275.643980:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761275.643983:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761275.643984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761275.643987:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507470464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761275.644007:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1253 00000100:00080000:2.0:1789761275.644013:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761275.644017:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507470592:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1789761275.644020:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761275.644028:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761275.644040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761275.644045:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507470848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761275.644051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761275/real 1789761275] req@ffff8d25c8020e00 x1876699507470592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761275.644071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507470592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761275.644078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761275.644081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761275.644083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507470592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761275.644088:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507470720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761275.644100:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761275.644106:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761275/real 1789761275] req@ffff8d25c8021500 x1876699507470720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761275.644113:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507470720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761275.644118:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761275.644120:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761275.644122:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507470720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761275.644129:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507470848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761275.644139:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761275.644146:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761275/real 1789761275] req@ffff8d25c8021c00 x1876699507470848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761275.644154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507470848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761300 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761275.644159:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761275.644160:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761275.644162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507470848:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761275.707586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761276.731505:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761277.755572:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761278.779502:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761279.803566:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761280.763511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761280.763522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761280.763526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761280.763538:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1258 00000100:00080000:2.0:1789761280.763551:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761280.763560:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761280.763614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507470976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761280.763641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761280.763644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761280.763646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761280.763650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1258 00000100:00080000:2.0:1789761280.763654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761280.763658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761280.763668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761280.763682:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507471104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761280.763684:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507470976:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761280.763699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761280.763702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761280.763703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761280.763706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1258 00000100:00080000:2.0:1789761280.763708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761280.763711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761280.763714:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761280.763722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507471232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761280.763729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761280.763732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761280.763733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761280.763736:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1258 00000100:00080000:2.0:1789761280.763738:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761280.763741:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761280.763745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761280.763751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507471360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761280.763763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761280.763765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761280.763766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761280.763769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1258 00000100:00080000:2.0:1789761280.763772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761280.763775:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761280.763779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761280.763783:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761280.763790:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507471488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761280.763797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761280.763800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761280.763801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761280.763804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1258 00000100:00080000:2.0:1789761280.763807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761280.763809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761280.763812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761280.763820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507471616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761280.763837:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761280/real 1789761280] req@ffff8d25c8021880 x1876699507470976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761280.763858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507470976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761280.763870:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761280.763876:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761280.763881:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507470976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761280.763910:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761280.763932:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761280.763944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761280/real 1789761280] req@ffff8d25c8023480 x1876699507471104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761280.763971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507471104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761280.763980:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761280.763984:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761280.763987:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761280.763994:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761280.764013:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761280.764024:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761280/real 1789761280] req@ffff8d25c8020000 x1876699507471232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761280.764035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507471232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761280.764042:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761280.764045:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761280.764049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761280.764054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761280.764070:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761280.764079:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761280/real 1789761280] req@ffff8d25c8023100 x1876699507471360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761280.764090:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507471360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761280.764098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761280.764102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761280.764106:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761280.764113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507471488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761280.764130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761280.764139:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761280/real 1789761280] req@ffff8d25c8022300 x1876699507471488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761280.764149:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507471488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761280.764157:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761280.764160:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761280.764163:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507471488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761280.764169:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507471616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761280.764185:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761280.764194:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761280/real 1789761280] req@ffff8d25c8022d80 x1876699507471616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761280.764205:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507471616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761305 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761280.764213:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761280.764216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761280.764219:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507471616:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761280.827529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761281.851547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761282.875530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761283.899536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761284.923549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761285.883476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761285.883493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761285.883497:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761285.883506:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1263 00000100:00080000:3.0:1789761285.883513:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761285.883519:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761285.883559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507471744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761285.883579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761285.883581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761285.883583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761285.883586:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1263 00000100:00080000:3.0:1789761285.883588:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761285.883591:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761285.883598:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761285.883607:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507471872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761285.883613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761285.883616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761285.883617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761285.883620:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1263 00000100:00080000:3.0:1789761285.883621:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761285.883623:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761285.883626:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761285.883632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507472000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761285.883639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761285.883642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761285.883644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761285.883647:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1263 00000100:00080000:3.0:1789761285.883648:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761285.883650:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761285.883655:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507471744:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789761285.883657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761285.883668:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507472128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761285.883679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761285.883683:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761285.883685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761285.883690:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1263 00000100:00080000:3.0:1789761285.883695:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761285.883700:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761285.883706:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761285.883714:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761285.883720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507472256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761285.883732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761285.883734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761285.883736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761285.883740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1263 00000100:00080000:3.0:1789761285.883744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761285.883747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761285.883751:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761285/real 1789761285] req@ffff8d25d1b0b100 x1876699507471744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761285.883753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761285.883766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507472384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761285.883793:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507471744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761285.883803:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761285.883807:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761285.883810:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507471744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761285.883816:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761285.883830:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761285.883839:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761285/real 1789761285] req@ffff8d25d1b0ad80 x1876699507471872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761285.883857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507471872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761285.883863:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761285.883865:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761285.883867:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507471872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761285.883870:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761285.883881:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761285.883887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761285/real 1789761285] req@ffff8d25d1b09c00 x1876699507472000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761285.883899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507472000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761285.883904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761285.883907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761285.883910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761285.883955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761285.883978:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761285.883989:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761285/real 1789761285] req@ffff8d25d1b09180 x1876699507472128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761285.883998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507472128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761285.884016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761285.884019:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761285.884023:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761285.884029:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507472256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761285.884048:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761285.884060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761285/real 1789761285] req@ffff8d25d1b0aa00 x1876699507472256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761285.884072:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507472256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761285.884086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761285.884090:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761285.884093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507472256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761285.884100:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507472384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761285.884120:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761285.884132:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761285/real 1789761285] req@ffff8d25d1b08a80 x1876699507472384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761285.884144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507472384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761310 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761285.884150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761285.884153:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761285.884156:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507472384:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761285.947544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761286.971604:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761287.995551:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761289.019552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761290.043547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761291.003482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761291.003490:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761291.003494:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761291.003502:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1268 00000100:00080000:3.0:1789761291.003510:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761291.003515:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761291.003545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507472512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761291.003571:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761291.003574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761291.003575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761291.003579:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1268 00000100:00080000:3.0:1789761291.003581:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761291.003585:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761291.003593:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761291.003601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507472512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761291.003604:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507472640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761291.003615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761291.003617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761291.003618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761291.003621:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1268 00000100:00080000:3.0:1789761291.003624:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761291.003627:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761291.003630:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761291.003638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507472768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761291.003645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761291.003647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761291.003649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761291.003651:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1268 00000100:00080000:3.0:1789761291.003655:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761291.003657:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761291.003661:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761291.003662:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761291.003673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507472896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761291.003684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761291.003688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761291.003690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761291.003695:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1268 00000100:00100000:0.0:1789761291.003699:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffff8d25d1b08380 x1876699507472512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761291.003701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761291.003711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761291.003717:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761291.003718:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507472512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761291.003729:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761291.003735:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761291.003737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507473024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761291.003739:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507472512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761291.003754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761291.003758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761291.003761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761291.003764:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761291.003769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1268 00000100:00080000:3.0:1789761291.003776:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761291.003781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761291.003786:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761291.003791:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761291.003797:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffff8d25d1b0a300 x1876699507472640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761291.003813:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507472640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761291.003814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507473152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761291.003829:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761291.003834:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761291.003840:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761291.003850:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761291.003868:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761291.003895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffff8d25d1b0b480 x1876699507472768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761291.003909:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507472768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761291.003918:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761291.003921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761291.003924:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761291.003931:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761291.003951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761291.003963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffff8d25d1b09500 x1876699507472896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761291.003974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507472896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761291.003979:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761291.003981:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761291.003984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507472896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761291.003987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761291.004002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761291.004013:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffff8d25d1b08000 x1876699507473024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761291.004025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507473024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761291.004033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761291.004037:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761291.004040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761291.004063:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761291.004079:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761291.004087:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761291/real 1789761291] req@ffff8d25d1b08380 x1876699507473152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761291.004100:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507473152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761316 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761291.004105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761291.004107:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761291.004109:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473152:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761291.067544:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761292.091519:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761293.115577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761294.139512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761295.163559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761296.123509:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.123517:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761296.123521:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761296.123529:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1273 00000100:00080000:0.0:1789761296.123537:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.123543:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761296.123574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507473280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761296.123597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.123600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761296.123601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761296.123604:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1273 00000100:00080000:0.0:1789761296.123606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.123608:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761296.123615:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761296.123625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507473408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761296.123634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507473280:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761296.123635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.123642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761296.123645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761296.123649:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1273 00000100:00080000:0.0:1789761296.123651:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.123653:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761296.123656:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761296.123663:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507473536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761296.123670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.123673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761296.123674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761296.123676:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1273 00000100:00080000:0.0:1789761296.123679:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.123681:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761296.123684:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761296.123692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507473664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761296.123700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761296.123701:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761296.123705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761296.123707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761296.123711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1273 00000100:00080000:0.0:1789761296.123715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.123717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761296.123721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761296.123728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507473792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761296.123735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761296.123737:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761296.123740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761296.123741:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffff8d25d25bb480 x1876699507473280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761296.123745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1273 00000100:00080000:0.0:1789761296.123750:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761296.123756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761296.123762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761296.123769:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507473280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761296.123772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507473920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761296.123794:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761296.123799:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761296.123803:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507473280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761296.123825:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507473408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761296.123863:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761296.123878:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffff8d25d25b8e00 x1876699507473408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761296.123892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507473408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761296.123902:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761296.123905:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761296.123909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507473408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761296.123916:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507473536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761296.123933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761296.123942:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffff8d25d25b9c00 x1876699507473536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761296.123950:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507473536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761296.123955:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761296.123957:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761296.123960:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507473536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761296.123963:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507473664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761296.123974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761296.123979:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffff8d25d25bb100 x1876699507473664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761296.123987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507473664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761296.123992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761296.123994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761296.123996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507473664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761296.123999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761296.124009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761296.124015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffff8d25d25b8700 x1876699507473792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761296.124027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507473792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761296.124033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761296.124034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761296.124036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761296.124039:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761296.124049:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761296.124055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761296/real 1789761296] req@ffff8d25d25bad80 x1876699507473920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761296.124063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507473920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761321 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761296.124068:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761296.124070:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761296.124071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507473920:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761296.187504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761297.211528:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761298.235508:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761299.259576:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761300.283587:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761301.243555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761301.243565:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761301.243571:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761301.243585:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1278 00000100:00080000:2.0:1789761301.243596:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761301.243606:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761301.243652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507474048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761301.243678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761301.243681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761301.243682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761301.243686:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1278 00000100:00080000:2.0:1789761301.243690:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761301.243694:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761301.243704:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761301.243717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507474176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761301.243724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761301.243727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761301.243729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761301.243733:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1278 00000100:00080000:2.0:1789761301.243735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761301.243739:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761301.243749:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761301.243759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507474304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761301.243768:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507474048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761301.243771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761301.243776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761301.243778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761301.243781:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1278 00000100:00080000:2.0:1789761301.243785:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761301.243789:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761301.243793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761301.243819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507474432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761301.243828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761301.243830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761301.243831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761301.243834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1278 00000100:00080000:2.0:1789761301.243837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761301.243840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761301.243843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761301.243848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507474560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761301.243854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761301.243856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761301.243858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761301.243861:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1278 00000100:00080000:2.0:1789761301.243863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761301.243865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761301.243868:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761301.243874:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507474688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761301.243926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761301.244069:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffff8d25c8020700 x1876699507474048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761301.244086:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507474048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761301.244136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761301.244141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761301.244145:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507474048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761301.244152:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761301.244171:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761301.244184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffff8d25c8023b80 x1876699507474176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761301.244198:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507474176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761301.244208:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761301.244211:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761301.244215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761301.244242:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761301.244260:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761301.244268:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffff8d25c8020380 x1876699507474304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761301.244284:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507474304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761301.244290:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761301.244292:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761301.244295:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761301.244299:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761301.244312:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761301.244319:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffff8d25c8023800 x1876699507474432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761301.244326:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507474432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761301.244332:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761301.244334:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761301.244335:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761301.244339:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507474560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761301.244350:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761301.244356:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffff8d25c8022a00 x1876699507474560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761301.244363:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507474560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761301.244368:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761301.244370:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761301.244372:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507474560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761301.244375:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507474688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761301.244412:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761301.244422:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761301/real 1789761301] req@ffff8d25c8022680 x1876699507474688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761301.244431:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507474688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761326 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761301.244437:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761301.244439:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761301.244441:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507474688:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761301.307618:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761302.331600:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761303.355581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761304.379595:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761305.403551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761306.363540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761306.363548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761306.363551:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761306.363559:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1284 00000100:00080000:0.0:1789761306.363569:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761306.363574:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761306.363610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507474816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761306.363633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761306.363636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761306.363638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761306.363641:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1284 00000100:00080000:0.0:1789761306.363644:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761306.363648:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761306.363656:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761306.363667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507474944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761306.363667:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507474816:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761306.363681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761306.363683:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761306.363685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761306.363688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1284 00000100:00080000:0.0:1789761306.363690:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761306.363692:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761306.363695:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761306.363704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507475072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761306.363709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761306.363712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761306.363713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761306.363716:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1284 00000100:00080000:0.0:1789761306.363720:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761306.363723:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761306.363726:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761306.363730:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761306.363736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507475200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761306.363745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761306.363747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761306.363749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761306.363751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1284 00000100:00080000:0.0:1789761306.363753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761306.363756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761306.363758:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761306.363781:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffff8d25d25b9f80 x1876699507474816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761306.363785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507475328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761306.363807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761306.363810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761306.363811:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761306.363815:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507474816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761306.363816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1284 00000100:00080000:0.0:1789761306.363821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761306.363827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761306.363834:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761306.363836:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761306.363842:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761306.363844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507475456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761306.363848:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507474816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761306.363877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761306.363900:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761306.363914:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffff8d25d25bb800 x1876699507474944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761306.363928:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507474944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761306.363944:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761306.363948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761306.363951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507474944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761306.363956:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761306.363972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761306.363980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffff8d25d25b8380 x1876699507475072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761306.363988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507475072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761306.363993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761306.363996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761306.363998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761306.364001:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761306.364011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761306.364017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffff8d25d25b9180 x1876699507475200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761306.364024:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507475200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761306.364029:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761306.364031:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761306.364034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761306.364038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507475328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761306.364048:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761306.364054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffff8d25d25ba680 x1876699507475328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761306.364061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507475328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761306.364066:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761306.364068:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761306.364070:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507475328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761306.364073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507475456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761306.364083:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761306.364088:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761306/real 1789761306] req@ffff8d25d25bbb80 x1876699507475456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761306.364096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507475456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761331 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761306.364101:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761306.364102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761306.364104:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507475456:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761306.427542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761307.451505:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761308.475520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761309.499591:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761310.523569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761311.483497:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761311.483513:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761311.483518:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761311.483533:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1289 00000100:00080000:1.0:1789761311.483546:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761311.483557:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761311.483610:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507475584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761311.483643:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761311.483649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761311.483651:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761311.483657:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1289 00000100:00080000:1.0:1789761311.483662:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761311.483668:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761311.483683:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761311.483684:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507475584:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761311.483702:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507475712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761311.483715:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761311.483718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761311.483720:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761311.483736:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1289 00000100:00080000:1.0:1789761311.483739:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761311.483743:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761311.483748:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761311.483760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507475840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761311.483768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761311.483771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761311.483773:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761311.483775:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1289 00000100:00080000:1.0:1789761311.483778:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761311.483781:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761311.483785:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761311.483790:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761311.483797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507475968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761311.483806:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761311.483809:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761311.483811:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761311.483815:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1289 00000100:00080000:1.0:1789761311.483818:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761311.483821:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761311.483824:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761311.483836:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507476096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761311.483847:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761311.483848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761311/real 1789761311] req@ffff8d25c8169c00 x1876699507475584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761311.483855:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761311.483859:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761311.483869:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1289 00000100:00080000:1.0:1789761311.483874:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761311.483879:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761311.483884:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507475584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761311.483885:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761311.483900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789761311.483901:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507476224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761311.483906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761311.483914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507475584:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761311.483943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761311.483962:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761311.483973:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761311/real 1789761311] req@ffff8d25c8169880 x1876699507475712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761311.483982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507475712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761311.483987:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761311.483990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761311.483992:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761311.483995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761311.484006:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761311.484012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761311/real 1789761311] req@ffff8d25c8168a80 x1876699507475840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761311.484024:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507475840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761311.484029:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761311.484031:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761311.484033:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761311.484037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761311.484048:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761311.484053:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761311/real 1789761311] req@ffff8d25c816ad80 x1876699507475968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761311.484060:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507475968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761311.484065:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761311.484067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761311.484069:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507475968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761311.484072:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761311.484082:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761311.484089:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761311/real 1789761311] req@ffff8d25c816a300 x1876699507476096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761311.484100:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507476096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761311.484115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761311.484118:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761311.484122:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761311.484128:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761311.484148:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761311.484156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761311/real 1789761311] req@ffff8d25c816b480 x1876699507476224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761311.484164:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507476224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761336 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761311.484169:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761311.484171:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761311.484173:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476224:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761311.547567:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761312.571547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761313.595528:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761314.619524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761315.643513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761316.603504:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761316.603515:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761316.603519:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761316.603530:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1294 00000100:00080000:3.0:1789761316.603539:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761316.603547:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761316.603587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507476352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761316.603627:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761316.603632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761316.603634:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761316.603639:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1294 00000100:00080000:3.0:1789761316.603643:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761316.603649:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761316.603661:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761316.603675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507476480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761316.603678:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507476352:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761316.603700:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761316.603703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761316.603705:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761316.603708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1294 00000100:00080000:3.0:1789761316.603711:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761316.603714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761316.603719:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761316.603729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507476608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761316.603736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761316.603739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761316.603740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761316.603743:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1294 00000100:00080000:3.0:1789761316.603747:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761316.603750:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761316.603754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761316.603761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507476736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761316.603769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761316.603771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761316.603772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761316.603777:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1294 00000800:00000100:0.0:1789761316.603780:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761316.603781:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761316.603786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761316.603789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761316.603797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507476864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761316.603804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761316.603807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761316.603808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761316.603811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1294 00000100:00080000:3.0:1789761316.603813:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761316.603815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761316.603818:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761316.603836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761316/real 1789761316] req@ffff8d25d1b08a80 x1876699507476352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761316.603839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507476992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761316.603885:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507476352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761316.603900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761316.603905:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761316.603910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507476352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761316.603942:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507476480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761316.603965:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761316.603978:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761316/real 1789761316] req@ffff8d25d1b0aa00 x1876699507476480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761316.603991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507476480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761316.604001:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761316.604004:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761316.604008:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507476480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761316.604017:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507476608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761316.604033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761316.604043:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761316/real 1789761316] req@ffff8d25d1b09180 x1876699507476608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761316.604055:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507476608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761316.604062:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761316.604064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761316.604067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507476608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761316.604071:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507476736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761316.604086:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761316.604093:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761316/real 1789761316] req@ffff8d25d1b09c00 x1876699507476736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761316.604101:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507476736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761316.604107:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761316.604109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761316.604111:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507476736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761316.604114:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761316.604125:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761316.604130:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761316/real 1789761316] req@ffff8d25d1b0ad80 x1876699507476864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761316.604139:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507476864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761316.604144:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761316.604145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761316.604147:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761316.604151:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761316.604161:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761316.604167:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761316/real 1789761316] req@ffff8d25d1b0b100 x1876699507476992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761316.604175:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507476992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761341 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761316.604180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761316.604182:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761316.604183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507476992:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761316.667580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761317.691551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761318.715517:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761319.739523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761320.763497:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761321.723526:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761321.723536:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761321.723540:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761321.723550:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1299 00000100:00080000:3.0:1789761321.723559:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761321.723566:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761321.723603:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507477120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761321.723629:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761321.723634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761321.723636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761321.723641:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1299 00000100:00080000:3.0:1789761321.723658:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761321.723665:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761321.723675:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761321.723681:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507477120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761321.723693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507477248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761321.723707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761321.723712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761321.723714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761321.723719:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1299 00000100:00080000:3.0:1789761321.723724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761321.723729:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761321.723734:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761321.723736:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761321.723752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507477376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761321.723763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761321.723768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1789761321.723772:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761321/real 1789761321] req@ffff8d25d1b0bb80 x1876699507477120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761321.723773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761321.723784:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1299 00000100:00080000:3.0:1789761321.723793:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761321.723798:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507477120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761321.723799:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761321.723812:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761321.723815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1789761321.723816:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761321.723822:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507477120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761321.723835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507477504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761321.723844:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507477248:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761321.723852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761321.723861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1789761321.723866:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761321.723866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761321.723875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1299 00000100:00100000:0.0:1789761321.723880:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761321/real 1789761321] req@ffff8d25d1b09880 x1876699507477248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761321.723880:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761321.723890:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761321.723896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507477248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761321.723899:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761321.723909:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761321.723912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761321.723914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507477248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761321.723920:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507477632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761321.723921:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507477376:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761321.723939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761321.723942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761321.723948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761321.723951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761321.723953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761321/real 1789761321] req@ffff8d25d1b0a680 x1876699507477376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761321.723959:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1299 00000100:00100000:0.0:1789761321.723968:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507477376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761321.723968:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761321.723979:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761321.723981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761321.723987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789761321.723988:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761321.723990:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507477376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761321.724009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507477760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761321.724012:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507477504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761321.724034:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761321.724045:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761321/real 1789761321] req@ffff8d25d1b08e00 x1876699507477504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761321.724058:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507477504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761321.724067:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761321.724070:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761321.724074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507477504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761321.724080:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507477632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761321.724101:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761321.724112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761321/real 1789761321] req@ffff8d25d1b0b800 x1876699507477632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761321.724125:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507477632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761321.724134:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761321.724138:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761321.724141:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507477632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761321.724156:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507477760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761321.724184:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761321.724197:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761321/real 1789761321] req@ffff8d25d1b08700 x1876699507477760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761321.724218:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507477760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761346 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761321.724226:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761321.724228:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761321.724231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507477760:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761321.787494:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761322.811570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761323.835572:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761324.859497:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761325.883521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761326.843463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761326.843475:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761326.843479:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761326.843490:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1304 00000100:00080000:3.0:1789761326.843501:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761326.843508:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761326.843561:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507477888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761326.843595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761326.843599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761326.843602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761326.843619:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1304 00000100:00080000:3.0:1789761326.843625:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761326.843629:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761326.843641:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761326.843657:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507477888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761326.843657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507478016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761326.843683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761326.843686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761326.843688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761326.843692:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1304 00000100:00080000:3.0:1789761326.843695:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761326.843698:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761326.843705:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761326.843715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507478144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761326.843724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761326.843727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761326.843728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761326.843732:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1304 00000800:00000100:0.0:1789761326.843742:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761326.843744:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761326.843750:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761326.843756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761326.843766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507478272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761326.843775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761326.843779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761326.843780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761326.843787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1304 00000100:00100000:0.0:1789761326.843788:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761326/real 1789761326] req@ffff8d25d1b08700 x1876699507477888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761326.843798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761326.843810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761326.843811:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507477888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761326.843826:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761326.843828:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761326.843838:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761326.843842:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507477888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761326.843847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507478400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761326.843857:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761326.843860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761326.843862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761326.843867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1304 00000100:00100000:0.0:1789761326.843872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478016:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761326.843874:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761326.843883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761326.843888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761326.843896:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761326.843901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507478528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761326.843911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761326/real 1789761326] req@ffff8d25d1b0b800 x1876699507478016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761326.843924:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507478016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761326.843931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761326.843934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761326.843936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761326.843941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761326.843954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761326.843961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761326/real 1789761326] req@ffff8d25d1b08e00 x1876699507478144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761326.843968:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507478144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761326.843981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761326.843984:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761326.843986:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761326.843990:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761326.844003:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761326.844010:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761326/real 1789761326] req@ffff8d25d1b0a680 x1876699507478272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761326.844017:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507478272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761326.844039:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761326.844041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761326.844044:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761326.844048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507478400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761326.844061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761326.844067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761326/real 1789761326] req@ffff8d25d1b09880 x1876699507478400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761326.844075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507478400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761326.844080:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761326.844082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761326.844084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507478400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761326.844126:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507478528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761326.844140:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761326.844148:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761326/real 1789761326] req@ffff8d25d1b0bb80 x1876699507478528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761326.844156:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507478528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761351 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761326.844161:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761326.844163:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761326.844165:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507478528:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761326.907529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761327.931562:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761328.955542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761329.979577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761331.003517:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761331.963464:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761331.963476:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761331.963481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761331.963490:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1309 00000100:00080000:3.0:1789761331.963499:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761331.963508:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761331.963556:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507478656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761331.963599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761331.963604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761331.963606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761331.963611:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1309 00000100:00080000:3.0:1789761331.963615:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761331.963621:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761331.963631:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761331.963644:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507478656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761331.963651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507478784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761331.963664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761331.963668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761331.963670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761331.963675:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1309 00000100:00080000:3.0:1789761331.963679:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761331.963685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761331.963692:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761331.963705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507478912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761331.963715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761331.963720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761331.963722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761331.963726:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1309 00000100:00080000:3.0:1789761331.963731:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761331.963736:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761331.963743:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761331.963750:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761331.963760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507479040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761331.963777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761331.963781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789761331.963786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478912:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761331.963786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761331.963797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1309 00000100:00080000:3.0:1789761331.963803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761331.963808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761331.963813:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761331.963827:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761331.963829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507479168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761331.963845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761331.963849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761331.963852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761331.963857:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507479168:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761331.963866:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1309 00000100:00080000:3.0:1789761331.963873:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761331.963879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761331.963886:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761331.963899:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507479296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761331.963943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507479296:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761332.027549:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761333.051490:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761334.075529:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761335.035775:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761335.035834:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761335.035855:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761335.035907:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761335.035920:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761335.035932:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761335.035942:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761335.035952:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761335.036238:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761331/real 1789761335] req@ffff8d25d1b08a80 x1876699507479296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761335.036275:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507479296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761335.036291:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761335.036297:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761335.036303:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507479296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761335.036314:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761331/real 1789761335] req@ffff8d25d1b0aa00 x1876699507479168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761335.036331:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507479168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761335.036341:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761335.036345:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761335.036350:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507479168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761335.036360:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761331/real 1789761335] req@ffff8d25d1b09180 x1876699507479040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761335.036431:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507479040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761335.036443:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761335.036448:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761335.036457:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761335.036464:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761331/real 1789761335] req@ffff8d25d1b0ad80 x1876699507478784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761335.036480:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507478784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761335.036491:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761335.036497:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761335.036502:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761335.036511:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761331/real 1789761335] req@ffff8d25d1b09c00 x1876699507478912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761335.036526:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507478912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761335.036536:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761335.036540:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761335.036543:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507478912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761335.036567:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761331/real 1789761335] req@ffff8d25d1b0b100 x1876699507478656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761335.036581:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507478656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761356 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761335.036588:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761335.036592:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761335.036595:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507478656:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789761335.099535:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761336.123521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761337.083516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761337.083523:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761337.083527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761337.083548:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1314 00000100:00080000:2.0:1789761337.083556:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761337.083562:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761337.083594:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507479424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761337.083621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761337.083624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761337.083626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761337.083630:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1314 00000100:00080000:2.0:1789761337.083633:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761337.083638:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761337.083646:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761337.083657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507479552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761337.083666:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507479424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761337.083671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761337.083678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761337.083680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761337.083686:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1314 00000100:00080000:2.0:1789761337.083691:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761337.083697:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761337.083703:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761337.083718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507479680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761337.083729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761337.083734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789761337.083738:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761337.083739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761337.083747:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1314 00000100:00080000:2.0:1789761337.083753:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761337.083757:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761337.083763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761337.083785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507479808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761337.083785:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffff8d25c8020a80 x1876699507479424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761337.083825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761337.083829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761337.083833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761337.083833:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507479424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761337.083842:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1314 00000100:00080000:2.0:1789761337.083851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761337.083857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761337.083858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761337.083868:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789761337.083869:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761337.083877:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507479424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761337.083888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507479936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761337.083900:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761337.083905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761337.083906:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761337.083909:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761337.083919:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1314 00000100:00080000:2.0:1789761337.083924:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761337.083928:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761337.083934:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761337.083945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761337.083950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507480064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761337.083965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffff8d25c8021180 x1876699507479552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761337.083984:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507479552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761337.083994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761337.083999:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761337.084003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761337.084010:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761337.084034:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761337.084045:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffff8d25c8022680 x1876699507479680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761337.084056:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507479680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761337.084066:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761337.084070:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761337.084073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761337.084080:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761337.084100:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761337.084111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffff8d25c8022a00 x1876699507479808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761337.084125:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507479808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761337.084133:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761337.084137:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761337.084140:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507479808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761337.084147:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507479936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761337.084169:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761337.084181:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffff8d25c8023800 x1876699507479936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761337.084202:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507479936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761337.084210:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761337.084213:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761337.084216:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507479936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761337.084234:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507480064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761337.085153:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761337.085170:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761337/real 1789761337] req@ffff8d25c8020380 x1876699507480064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761337.085181:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507480064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761362 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761337.085228:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761337.085232:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761337.085236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507480064:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761337.147498:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761338.171540:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761339.195553:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761340.219570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761341.243556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761342.203453:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761342.203462:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761342.203466:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761342.203477:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1319 00000100:00080000:2.0:1789761342.203487:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761342.203508:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761342.203555:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507480192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761342.203585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761342.203588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761342.203590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761342.203594:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1319 00000100:00080000:2.0:1789761342.203598:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761342.203602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761342.203613:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761342.203625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507480192:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761342.203627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507480320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761342.203642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761342.203644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761342.203646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761342.203649:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1319 00000100:00080000:2.0:1789761342.203651:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761342.203655:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761342.203659:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761342.203667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507480448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761342.203674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761342.203677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761342.203678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761342.203680:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1319 00000100:00080000:2.0:1789761342.203695:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761342.203697:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761342.203701:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761342.203709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507480576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761342.203717:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761342.203722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761342.203726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761342.203727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761342.203731:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1319 00000100:00080000:2.0:1789761342.203735:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761342.203738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761342.203741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761342.203747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507480704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761342.203754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761342.203757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761342.203758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761342.203761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1319 00000100:00080000:2.0:1789761342.203764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761342.203767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761342.203770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761342.203780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507480832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761342.203781:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffff8d25c8023b80 x1876699507480192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761342.203815:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507480192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761342.203829:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761342.203834:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761342.203839:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507480192:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761342.203867:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507480320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761342.203892:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761342.203906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffff8d25c8020700 x1876699507480320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761342.203929:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507480320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761342.203939:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761342.203943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761342.203947:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507480320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761342.203953:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507480448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761342.203973:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761342.203985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffff8d25c8022d80 x1876699507480448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761342.203998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507480448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761342.204007:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761342.204010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761342.204014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507480448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761342.204020:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507480576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761342.204040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761342.204051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffff8d25c8022300 x1876699507480576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761342.204064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507480576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761342.204073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761342.204084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761342.204088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507480576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761342.204095:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507480704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761342.204115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761342.204126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffff8d25c8023100 x1876699507480704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761342.204139:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507480704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761342.204148:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761342.204151:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761342.204155:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507480704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761342.204162:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507480832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761342.204179:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761342.204189:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761342/real 1789761342] req@ffff8d25c8020000 x1876699507480832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761342.204203:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507480832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761367 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761342.204212:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761342.204215:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761342.204219:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507480832:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761342.267539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761343.291523:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761344.315513:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761345.339521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761346.363503:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761347.323478:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761347.323486:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761347.323490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761347.323498:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1325 00000100:00080000:3.0:1789761347.323506:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761347.323512:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761347.323542:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507480960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761347.323565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761347.323568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761347.323569:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761347.323573:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1325 00000100:00080000:3.0:1789761347.323575:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761347.323578:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761347.323585:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761347.323593:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507481088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761347.323602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761347.323607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761347.323608:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507480960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761347.323609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761347.323617:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1325 00000100:00080000:3.0:1789761347.323620:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761347.323623:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761347.323626:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761347.323635:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507481216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761347.323642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761347.323644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761347.323646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761347.323649:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1325 00000100:00080000:3.0:1789761347.323652:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761347.323654:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761347.323657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789761347.323668:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761347.323679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507481344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761347.323687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761347.323690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761347.323691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761347.323695:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1325 00000100:00080000:3.0:1789761347.323699:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761347.323702:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761347.323705:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffff8d25d1b08380 x1876699507480960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761347.323708:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761347.323722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507481472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761347.323726:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507480960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761347.323746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761347.323749:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761347.323749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761347.323753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761347.323756:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761347.323760:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1325 00000100:00100000:0.0:1789761347.323763:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507480960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761347.323764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761347.323772:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761347.323776:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761347.323783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507481600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761347.323790:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761347.323811:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761347.323820:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffff8d25d1b08000 x1876699507481088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.323828:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507481088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.323834:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.323836:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761347.323839:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761347.323844:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761347.323863:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761347.323875:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffff8d25d1b09500 x1876699507481216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.323889:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507481216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.323897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.323901:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761347.323904:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761347.323911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761347.323933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761347.323944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffff8d25d1b0b480 x1876699507481344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.323957:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507481344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.323965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.323969:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761347.323972:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761347.323979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507481472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761347.323999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761347.324010:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffff8d25d1b0a300 x1876699507481472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.324019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507481472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.324025:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.324027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761347.324029:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507481472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761347.324045:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507481600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761347.324059:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761347.324066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761347/real 1789761347] req@ffff8d25d1b0b100 x1876699507481600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761347.324073:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507481600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761372 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761347.324078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761347.324080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761347.324082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507481600:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761347.387564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761348.411539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761349.435563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761350.459552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761351.483544:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761352.443481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761352.443492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761352.443497:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761352.443508:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1330 00000100:00080000:3.0:1789761352.443518:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761352.443526:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761352.443571:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507481728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761352.443607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761352.443612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761352.443614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761352.443619:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1330 00000100:00080000:3.0:1789761352.443624:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761352.443630:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761352.443640:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761352.443648:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507481728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761352.443657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507481856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761352.443671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761352.443676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761352.443678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761352.443683:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1330 00000100:00080000:3.0:1789761352.443688:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761352.443693:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761352.443698:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761352.443711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507481984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761352.443721:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761352.443730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761352.443736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761352.443739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761352.443744:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1330 00000100:00080000:3.0:1789761352.443750:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761352.443755:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761352.443762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761352.443766:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761352/real 1789761352] req@ffff8d25d1b09c00 x1876699507481728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761352.443779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507482112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761352.443784:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507481728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761352.443804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761352.443807:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761352.443809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761352.443814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761352.443818:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761352.443822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1330 00000100:00080000:3.0:1789761352.443827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761352.443830:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507481728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761352.443832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761352.443843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761352.443857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507482240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761352.443867:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761352.443871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761352.443877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761352.443880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761352.443885:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1330 00000100:00080000:3.0:1789761352.443891:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761352.443896:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761352.443902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761352.443903:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761352.443919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507482368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761352.443923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761352/real 1789761352] req@ffff8d25d1b0ad80 x1876699507481856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761352.443959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507481856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761352.443970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761352.443974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761352.443978:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761352.443986:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761352.444014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761352.444027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761352/real 1789761352] req@ffff8d25d1b09180 x1876699507481984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761352.444040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507481984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761352.444047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761352.444049:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761352.444051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507481984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761352.444056:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761352.444071:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761352.444078:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761352/real 1789761352] req@ffff8d25d1b0aa00 x1876699507482112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761352.444085:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507482112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761352.444091:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761352.444093:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761352.444095:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761352.444109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507482240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761352.444119:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761352.444125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761352/real 1789761352] req@ffff8d25d1b08a80 x1876699507482240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761352.444137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507482240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761352.444142:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761352.444144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761352.444146:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507482240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761352.444149:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507482368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761352.444159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761352.444165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761352/real 1789761352] req@ffff8d25d1b0bb80 x1876699507482368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761352.444172:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507482368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761377 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761352.444177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761352.444179:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761352.444180:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507482368:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761352.507519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761353.531529:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761354.555524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761355.579691:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761356.603507:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761357.563469:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761357.563477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761357.563481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761357.563490:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1335 00000100:00080000:3.0:1789761357.563498:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761357.563504:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761357.563536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507482496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761357.563558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761357.563561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761357.563563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761357.563566:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1335 00000100:00080000:3.0:1789761357.563569:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761357.563571:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761357.563579:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761357.563587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507482624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761357.563594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761357.563597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761357.563599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761357.563601:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1335 00000100:00080000:3.0:1789761357.563604:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761357.563607:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761357.563610:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761357.563618:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507482496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761357.563619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507482752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761357.563633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761357.563636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761357.563637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761357.563640:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1335 00000100:00080000:3.0:1789761357.563643:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761357.563645:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761357.563648:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761357.563664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507482880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761357.563672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761357.563675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761357.563677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761357.563678:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1335 00000100:00080000:3.0:1789761357.563684:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761357.563688:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761357.563690:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761357.563693:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761357.563702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507483008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761357.563714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761357.563716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761357.563717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761357.563720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1335 00000100:00080000:3.0:1789761357.563727:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761357.563730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761357.563734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761357.563736:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761357/real 1789761357] req@ffff8d25d1b09880 x1876699507482496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761357.563747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507483136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761357.563761:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507482496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761357.563774:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761357.563778:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761357.563783:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507482496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761357.563791:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761357.563814:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761357.563827:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761357/real 1789761357] req@ffff8d25d1b0a680 x1876699507482624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761357.563840:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507482624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761357.563848:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761357.563852:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761357.563855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761357.563895:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761357.563916:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761357.563928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761357/real 1789761357] req@ffff8d25d1b08e00 x1876699507482752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761357.563941:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507482752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761357.563948:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761357.563952:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761357.563955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761357.563962:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761357.563981:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761357.563992:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761357/real 1789761357] req@ffff8d25d1b0b800 x1876699507482880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761357.564003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507482880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761357.564012:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761357.564014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761357.564018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507482880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761357.564024:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761357.564042:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761357.564054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761357/real 1789761357] req@ffff8d25d1b08700 x1876699507483008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761357.564065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507483008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761357.564073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761357.564076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761357.564079:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761357.564085:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761357.564103:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761357.564114:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761357/real 1789761357] req@ffff8d25d1b0bb80 x1876699507483136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761357.564126:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507483136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761382 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761357.564134:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761357.564137:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761357.564140:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483136:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761357.627551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761358.651510:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761359.675498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761360.699537:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761361.723540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761362.683492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761362.683503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761362.683507:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761362.683518:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1340 00000100:00080000:0.0:1789761362.683530:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761362.683538:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761362.683581:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507483264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761362.683613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761362.683616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761362.683618:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761362.683621:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1340 00000100:00080000:0.0:1789761362.683623:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761362.683627:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761362.683637:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761362.683648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507483392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761362.683657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761362.683667:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507483264:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761362.683668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761362.683671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761362.683682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1340 00000100:00080000:0.0:1789761362.683687:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761362.683690:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761362.683693:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761362.683701:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507483520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761362.683708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761362.683711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761362.683712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761362.683715:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1340 00000100:00080000:0.0:1789761362.683717:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761362.683719:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761362.683723:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761362.683732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507483648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761362.683740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761362.683742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761362.683744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761362.683746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1340 00000100:00080000:0.0:1789761362.683756:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761362.683760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761362.683763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761362.683772:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761362.683778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507483776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761362.683787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761362.683790:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761362.683791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761362.683796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1340 00000100:00080000:0.0:1789761362.683806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761362.683809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761362.683812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761362.683821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507483904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761362.683835:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761362/real 1789761362] req@ffff8d25d25b8000 x1876699507483264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761362.683859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507483264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761362.683872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761362.683877:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761362.683882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507483264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761362.683911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507483392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761362.683950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761362.683967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761362/real 1789761362] req@ffff8d25d25ba300 x1876699507483392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761362.683982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507483392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761362.683992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761362.683996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761362.684001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507483392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761362.684008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507483520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761362.684031:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761362.684043:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761362/real 1789761362] req@ffff8d25d25b8a80 x1876699507483520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761362.684057:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507483520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761362.684066:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761362.684070:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761362.684075:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507483520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761362.684082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507483648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761362.684102:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761362.684113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761362/real 1789761362] req@ffff8d25d25bbb80 x1876699507483648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761362.684126:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507483648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761362.684136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761362.684140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761362.684144:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507483648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761362.684151:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761362.684172:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761362.684184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761362/real 1789761362] req@ffff8d25d25ba680 x1876699507483776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761362.684198:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507483776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761362.684207:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761362.684211:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761362.684215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761362.684222:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761362.684245:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761362.684258:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761362/real 1789761362] req@ffff8d25d25b9180 x1876699507483904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761362.684272:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507483904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761387 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761362.684280:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761362.684284:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761362.684289:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507483904:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761362.747580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761363.771564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761364.795514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761365.819583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761366.843622:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761367.803468:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761367.803483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761367.803488:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761367.803502:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1345 00000100:00080000:0.0:1789761367.803515:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761367.803525:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761367.803584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507484032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761367.803626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761367.803631:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761367.803633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761367.803639:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1345 00000100:00080000:0.0:1789761367.803643:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761367.803648:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761367.803660:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761367.803676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507484160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761367.803689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761367.803694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507484032:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761367.803695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761367.803699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761367.803708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1345 00000100:00080000:0.0:1789761367.803712:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761367.803717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761367.803723:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761367.803736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507484288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761367.803747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761367.803751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761367.803753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761367.803758:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1345 00000100:00080000:0.0:1789761367.803771:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761367.803776:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761367.803782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761367.803798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507484416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761367.803798:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761367.803813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761367.803818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761367.803820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761367.803825:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1345 00000100:00080000:0.0:1789761367.803830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761367.803835:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761367.803840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761367.803853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507484544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761367.803859:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761367/real 1789761367] req@ffff8d25d25b8380 x1876699507484032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761367.803872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761367.803877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761367.803879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761367.803887:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1345 00000100:00080000:0.0:1789761367.803893:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761367.803900:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761367.803901:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507484032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761367.803909:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761367.803923:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789761367.803927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507484672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761367.803932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761367.803939:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507484032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761367.803976:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761367.804005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761367.804020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761367/real 1789761367] req@ffff8d25d25bb800 x1876699507484160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761367.804035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507484160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761367.804045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761367.804048:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761367.804052:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761367.804059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761367.804084:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761367.804097:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761367/real 1789761367] req@ffff8d25d25b9f80 x1876699507484288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761367.804109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507484288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761367.804117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761367.804121:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761367.804124:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761367.804131:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761367.804153:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761367.804165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761367/real 1789761367] req@ffff8d25d25bad80 x1876699507484416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761367.804177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507484416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761367.804186:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761367.804189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761367.804193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761367.804199:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507484544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761367.804221:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761367.804234:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761367/real 1789761367] req@ffff8d25d25b8700 x1876699507484544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761367.804254:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507484544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761367.804263:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761367.804266:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761367.804270:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507484544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761367.804276:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507484672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761367.804299:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761367.804328:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761367/real 1789761367] req@ffff8d25d25bb100 x1876699507484672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761367.804342:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507484672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761392 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761367.804350:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761367.804353:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761367.804356:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507484672:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761367.867545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761368.891556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761369.915520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761370.939542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761371.963519:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761372.923572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761372.923578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761372.923582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761372.923591:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1350 00000100:00080000:0.0:1789761372.923600:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761372.923607:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761372.923636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507484800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761372.923659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761372.923663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761372.923664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761372.923667:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1350 00000100:00080000:0.0:1789761372.923670:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761372.923674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761372.923682:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761372.923697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507484928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761372.923708:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507484800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761372.923720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761372.923726:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761372.923729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761372.923737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1350 00000100:00080000:0.0:1789761372.923743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761372.923749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761372.923754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761372.923769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507485056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761372.923777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761372.923780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761372.923781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761372.923785:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1350 00000100:00080000:0.0:1789761372.923789:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761372.923791:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761372.923795:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761372.923804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507485184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761372.923817:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761372.923820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761372.923824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761372.923826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761372.923834:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1350 00000100:00080000:0.0:1789761372.923839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761372.923844:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761372.923850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761372.923871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507485312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761372.923878:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761372/real 1789761372] req@ffff8d25d25b9c00 x1876699507484800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761372.923939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761372.923942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761372.923948:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761372.923948:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507484800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761372.923962:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1350 00000100:00080000:0.0:1789761372.923974:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761372.923980:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761372.923986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761372.923990:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761372.923991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507484800:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761372.924003:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761372.924017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507485440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761372.924031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761372.924063:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761372.924074:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761372/real 1789761372] req@ffff8d25d25b8e00 x1876699507484928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761372.924083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507484928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761372.924089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761372.924092:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761372.924096:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507484928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761372.924101:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761372.924114:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761372.924121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761372/real 1789761372] req@ffff8d25d25bb480 x1876699507485056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761372.924131:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507485056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761372.924136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761372.924138:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761372.924140:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761372.924144:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761372.924160:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761372.924171:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761372/real 1789761372] req@ffff8d25d25bb100 x1876699507485184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761372.924180:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507485184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761372.924186:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761372.924188:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761372.924190:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761372.924194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507485312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761372.924206:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761372.924216:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761372/real 1789761372] req@ffff8d25d25b8700 x1876699507485312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761372.924224:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507485312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761372.924229:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761372.924231:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761372.924234:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507485312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761372.924252:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507485440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761372.924286:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761372.924308:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761372/real 1789761372] req@ffff8d25d25bad80 x1876699507485440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761372.924331:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507485440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761397 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761372.924340:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761372.924344:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761372.924347:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507485440:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761372.987556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761374.011518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761375.035559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761376.059489:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761377.083515:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761378.043599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761378.043608:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761378.043612:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761378.043621:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1355 00000100:00080000:3.0:1789761378.043629:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761378.043635:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761378.043665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507485568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761378.043690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761378.043693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761378.043695:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761378.043699:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1355 00000100:00080000:3.0:1789761378.043702:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761378.043706:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761378.043714:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761378.043718:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507485568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761378.043725:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507485696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761378.043734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761378.043737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761378.043739:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761378.043742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1355 00000100:00080000:3.0:1789761378.043745:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761378.043748:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761378.043751:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761378.043759:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507485824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761378.043766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761378.043769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761378.043770:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761378.043776:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761378.043776:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1355 00000100:00080000:3.0:1789761378.043783:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761378.043786:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761378.043791:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761378.043799:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507485952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761378.043807:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761378.043810:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761378.043811:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761378.043813:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1355 00000100:00080000:3.0:1789761378.043816:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761378.043819:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761378.043822:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761378.043828:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffff8d25d1b08a80 x1876699507485568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761378.043831:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507486080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761378.043857:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761378.043861:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761378.043862:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761378.043865:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507485568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761378.043869:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1355 00000100:00080000:3.0:1789761378.043874:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761378.043880:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789761378.043883:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761378.043885:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789761378.043890:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761378.043894:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507485568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761378.043897:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507486208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761378.043919:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761378.043938:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761378.043948:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffff8d25d1b0aa00 x1876699507485696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761378.043956:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507485696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761378.043963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761378.043965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761378.043968:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761378.043972:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761378.043983:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761378.043989:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffff8d25d1b09180 x1876699507485824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761378.043997:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507485824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761378.044002:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761378.044004:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761378.044006:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761378.044010:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761378.044020:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761378.044026:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffff8d25d1b0ad80 x1876699507485952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761378.044034:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507485952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761378.044039:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761378.044040:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761378.044043:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507485952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761378.044047:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761378.044057:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761378.044062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffff8d25d1b09c00 x1876699507486080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761378.044069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507486080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761378.044075:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761378.044077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761378.044078:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761378.044082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761378.044091:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761378.044097:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761378/real 1789761378] req@ffff8d25d1b0b100 x1876699507486208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761378.044104:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507486208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761403 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761378.044109:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761378.044111:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761378.044113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486208:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761378.107568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761379.131559:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761380.155534:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761381.179518:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761382.203483:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761383.163466:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761383.163473:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761383.163477:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761383.163486:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1360 00000100:00080000:0.0:1789761383.163495:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761383.163503:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761383.163543:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507486336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761383.163571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761383.163575:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761383.163578:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761383.163584:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1360 00000100:00080000:0.0:1789761383.163589:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761383.163594:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761383.163599:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507486336:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761383.163607:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761383.163628:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507486464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761383.163640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761383.163644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761383.163647:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761383.163651:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1360 00000800:00000100:2.0:1789761383.163655:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761383.163658:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761383.163665:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761383.163672:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761383.163684:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507486592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761383.163694:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffff8d25d25b9f80 x1876699507486336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761383.163696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761383.163707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761383.163712:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761383.163719:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1360 00000100:00100000:2.0:1789761383.163721:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507486336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761383.163724:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761383.163732:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789761383.163736:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789761383.163738:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1789761383.163741:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761383.163746:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507486336:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761383.163753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507486720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761383.163762:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761383.163765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761383.163768:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761383.163768:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507486464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761383.163777:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1360 00000100:00080000:0.0:1789761383.163781:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761383.163784:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761383.163787:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761383.163796:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761383.163800:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507486848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761383.163809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761383.163811:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffff8d25d25bb800 x1876699507486464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761383.163813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761383.163817:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761383.163826:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1360 00000100:00080000:0.0:1789761383.163830:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761383.163834:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507486464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761383.163836:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761383.163843:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789761383.163845:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761383.163850:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761383.163853:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507486976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761383.163856:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507486464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761383.163867:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507486592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761383.163890:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761383.163903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffff8d25d25b8380 x1876699507486592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761383.163923:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507486592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761383.163933:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761383.163936:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761383.163940:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507486592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761383.163946:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507486720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761383.163968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761383.163980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffff8d25d25b9180 x1876699507486720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761383.163988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507486720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761383.164000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761383.164002:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761383.164004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507486720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761383.164018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761383.164038:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761383.164051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffff8d25d25ba680 x1876699507486848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761383.164064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507486848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761383.164079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761383.164083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761383.164087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761383.164094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761383.164115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761383.164124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761383/real 1789761383] req@ffff8d25d25bbb80 x1876699507486976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761383.164132:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507486976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761408 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761383.164146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761383.164148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761383.164151:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507486976:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761383.227517:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761384.251546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761385.275502:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761386.299541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761387.323689:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761388.283507:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761388.283517:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761388.283522:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761388.283532:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1365 00000100:00080000:3.0:1789761388.283541:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761388.283549:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761388.283587:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507487104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761388.283620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761388.283624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761388.283627:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761388.283632:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1365 00000100:00080000:3.0:1789761388.283637:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761388.283642:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761388.283652:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761388.283661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507487104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761388.283670:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507487232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761388.283686:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761388.283691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761388.283693:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761388.283698:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1365 00000100:00080000:3.0:1789761388.283714:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761388.283721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761388.283727:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761388.283729:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761388.283745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507487360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761388.283758:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761388.283762:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761388.283764:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761388.283767:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1365 00000100:00080000:3.0:1789761388.283771:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761388.283775:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761388.283780:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffff8d25d1b0a300 x1876699507487104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761388.283780:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761388.283799:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507487488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761388.283802:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507487104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761388.283832:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761388.283834:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761388.283837:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761388.283843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789761388.283845:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507487104:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761388.283848:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761388.283855:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1365 00000100:00080000:3.0:1789761388.283860:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761388.283866:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761388.283870:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507487232:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789761388.283875:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761388.283892:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507487616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761388.283896:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761388.283909:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761388.283911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffff8d25d1b0b480 x1876699507487232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761388.283913:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761388.283917:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761388.283925:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1365 00000100:00080000:3.0:1789761388.283933:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761388.283940:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761388.283945:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507487232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761388.283948:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789761388.283964:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761388.283969:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507487744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761388.283970:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761388.283977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507487232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761388.283988:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507487360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761388.284011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761388.284022:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffff8d25d1b09500 x1876699507487360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761388.284030:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507487360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761388.284037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761388.284041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761388.284045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507487360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761388.284051:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507487488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761388.284071:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761388.284084:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffff8d25d1b08000 x1876699507487488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761388.284097:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507487488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761388.284106:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761388.284110:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761388.284113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507487488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761388.284133:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507487616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761388.284170:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761388.284183:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffff8d25d1b08380 x1876699507487616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761388.284197:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507487616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761388.284204:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761388.284206:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761388.284209:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507487616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761388.284213:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507487744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761388.284228:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761388.284236:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761388/real 1789761388] req@ffff8d25d1b0a300 x1876699507487744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761388.284243:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507487744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761413 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761388.284249:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761388.284251:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761388.284254:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507487744:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761388.347587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761389.371531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761390.395523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761391.419495:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761392.443556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761393.403514:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761393.403525:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761393.403531:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761393.403545:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1371 00000100:00080000:3.0:1789761393.403559:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761393.403568:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761393.403616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507487872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761393.403642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761393.403645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761393.403647:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761393.403650:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1371 00000100:00080000:3.0:1789761393.403653:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761393.403657:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761393.403668:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761393.403679:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507488000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761393.403689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761393.403691:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507487872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761393.403693:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761393.403695:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761393.403706:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1371 00000100:00080000:3.0:1789761393.403713:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761393.403716:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761393.403720:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761393.403728:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507488128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761393.403734:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761393.403737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761393.403739:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761393.403742:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1371 00000100:00080000:3.0:1789761393.403744:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761393.403746:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761393.403749:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761393.403759:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507488256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761393.403766:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761393.403769:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761393.403771:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761393.403773:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1371 00000100:00080000:3.0:1789761393.403776:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761393.403778:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761393.403781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761393.403785:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761393.403793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507488384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761393.403805:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761393.403808:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761393.403810:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761393.403815:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1371 00000100:00080000:3.0:1789761393.403819:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761393.403823:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761393.403829:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761393.403840:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507488512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761393.403853:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761393/real 1789761393] req@ffff8d25d1b0b100 x1876699507487872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761393.403870:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507487872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761393.403880:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761393.403883:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761393.403886:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507487872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761393.403907:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761393.403925:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761393.403934:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761393/real 1789761393] req@ffff8d25d1b09c00 x1876699507488000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761393.403943:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507488000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761393.403949:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761393.403952:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761393.403954:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761393.403958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761393.403969:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761393.403976:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761393/real 1789761393] req@ffff8d25d1b0ad80 x1876699507488128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761393.403983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507488128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761393.403999:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761393.404000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761393.404002:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761393.404006:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761393.404017:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761393.404023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761393/real 1789761393] req@ffff8d25d1b09180 x1876699507488256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761393.404031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507488256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761393.404037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761393.404039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761393.404040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761393.404045:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507488384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761393.404074:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761393.404086:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761393/real 1789761393] req@ffff8d25d1b0aa00 x1876699507488384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761393.404099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507488384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761393.404132:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761393.404136:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761393.404139:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507488384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761393.404144:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507488512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761393.404161:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761393.404169:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761393/real 1789761393] req@ffff8d25d1b08a80 x1876699507488512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761393.404177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507488512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761418 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761393.404183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761393.404185:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761393.404187:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507488512:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761393.467558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761394.491651:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761395.515557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761396.539582:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761397.563551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761398.523463:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761398.523474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761398.523478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761398.523489:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1376 00000100:00080000:0.0:1789761398.523499:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761398.523506:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761398.523536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507488640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761398.523557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761398.523561:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761398.523563:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761398.523568:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1376 00000100:00080000:0.0:1789761398.523571:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761398.523576:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761398.523585:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761398.523590:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507488640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761398.523598:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507488768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761398.523608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761398.523611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761398.523612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761398.523616:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1376 00000100:00080000:0.0:1789761398.523619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761398.523622:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761398.523626:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761398.523636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507488896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761398.523645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761398.523648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761398.523650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761398.523654:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1376 00000100:00080000:0.0:1789761398.523657:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761398.523661:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761398.523666:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761398.523678:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507489024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761398.523687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789761398.523689:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488768:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761398.523690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761398.523692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761398.523698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1376 00000100:00080000:0.0:1789761398.523701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761398.523704:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761398.523709:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761398.523721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507489152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761398.523721:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488896:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761398.523732:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761398.523735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761398.523737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761398.523742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1376 00000100:00100000:2.0:1789761398.523743:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489024:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1789761398.523745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761398.523758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761398.523763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761398.523770:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507489152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761398.523774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507489280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761398.523792:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507489280:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761398.587558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761399.611555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761400.635574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761401.595682:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1789761401.595740:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761401.595761:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761401.595812:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761401.595828:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761401.595843:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761401.595848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761398/real 1789761401] req@ffff8d25d25ba300 x1876699507488768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:2.0:1789761401.595861:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761401.595876:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761401.595897:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507488768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761401.595913:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761401.595919:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761401.595923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761401.595933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761398/real 1789761401] req@ffff8d25d25b8000 x1876699507488896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761401.595947:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507488896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761401.595955:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761401.595966:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761401.595970:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507488896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761401.595978:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761398/real 1789761401] req@ffff8d25d25bbb80 x1876699507489024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761401.595991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507489024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761401.595999:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761401.596003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761401.596006:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761401.596014:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761398/real 1789761401] req@ffff8d25d25b8a80 x1876699507488640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761401.596027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507488640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761401.596035:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761401.596039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761401.596042:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507488640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761401.596093:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761398/real 1789761401] req@ffff8d25d25b9180 x1876699507489280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761401.596108:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507489280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761401.596116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761401.596120:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761401.596123:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507489280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761401.596130:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761398/real 1789761401] req@ffff8d25d25ba680 x1876699507489152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761401.596144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507489152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761423 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761401.596152:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761401.596156:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761401.596159:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507489152:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761401.659505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761402.683544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761403.643480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761403.643491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761403.643496:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761403.643506:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1381 00000100:00080000:0.0:1789761403.643525:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761403.643534:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761403.643583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507489408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761403.643616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761403.643620:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761403.643622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761403.643628:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1381 00000100:00080000:0.0:1789761403.643632:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761403.643637:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761403.643644:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507489408:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761403.643647:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761403.643664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507489536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761403.643677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761403.643681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761403.643683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761403.643688:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1381 00000100:00080000:0.0:1789761403.643692:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761403.643696:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761403.643701:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761403.643723:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761403.643725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507489664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761403.643737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761403.643741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761403.643743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761403.643747:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1381 00000100:00080000:0.0:1789761403.643750:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761403.643755:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761403.643760:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761403.643768:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761403/real 1789761403] req@ffff8d25d25b8380 x1876699507489408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761403.643774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507489792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761403.643795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761403.643801:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761403.643803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761403.643814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1381 00000100:00080000:0.0:1789761403.643819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761403.643820:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507489408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761403.643828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761403.643839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761403.643841:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761403.643850:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761403.643855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507489408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761403.643859:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507489920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761403.643870:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761403.643874:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761403.643875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761403.643880:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1381 00000100:00080000:0.0:1789761403.643884:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761403.643885:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489536:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761403.643896:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761403.643905:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761403.643918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761403.643926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507490048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761403.643935:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761403/real 1789761403] req@ffff8d25d25bb800 x1876699507489536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761403.643973:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507489536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761403.643983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761403.643987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761403.643991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761403.643999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761403.644021:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761403.644057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761403/real 1789761403] req@ffff8d25d25b9f80 x1876699507489664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761403.644078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507489664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761403.644086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761403.644089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761403.644093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761403.644099:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761403.644120:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761403.644132:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761403/real 1789761403] req@ffff8d25d25bad80 x1876699507489792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761403.644143:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507489792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761403.644152:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761403.644156:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761403.644160:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507489792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761403.644166:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507489920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761403.644187:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761403.644199:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761403/real 1789761403] req@ffff8d25d25b8700 x1876699507489920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761403.644210:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507489920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761403.644221:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761403.644223:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761403.644225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507489920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761403.644240:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507490048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761403.644262:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761403.644273:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761403/real 1789761403] req@ffff8d25d25bb100 x1876699507490048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761403.644285:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507490048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761428 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761403.644294:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761403.644297:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761403.644300:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507490048:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761403.707587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761404.731553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761405.755589:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761406.779546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761407.803519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761408.763503:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761408.763514:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761408.763518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761408.763529:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1386 00000100:00080000:0.0:1789761408.763539:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761408.763547:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761408.763579:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507490176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761408.763599:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761408.763602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761408.763603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761408.763606:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1386 00000100:00080000:0.0:1789761408.763609:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761408.763613:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761408.763621:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761408.763634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507490304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761408.763634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507490176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761408.763651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761408.763653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761408.763654:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761408.763656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1386 00000100:00080000:0.0:1789761408.763659:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761408.763662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761408.763664:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761408.763673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507490432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761408.763679:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761408.763682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761408.763684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761408.763686:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1386 00000100:00080000:0.0:1789761408.763688:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761408.763690:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761408.763696:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761408.763706:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761408.763712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507490560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761408.763725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761408.763729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761408.763731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761408.763735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1386 00000100:00080000:0.0:1789761408.763740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761408.763745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761408.763751:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761408/real 1789761408] req@ffff8d25d25bb480 x1876699507490176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761408.763752:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761408.763773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507490688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761408.763779:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507490176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761408.763800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761408.763805:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761408.763806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761408.763810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761408.763815:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761408.763819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1386 00000100:00100000:3.0:1789761408.763825:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507490176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761408.763827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761408.763836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761408.763842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761408.763852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507490816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761408.763863:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507490304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761408.763896:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761408.763913:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761408/real 1789761408] req@ffff8d25d25b8e00 x1876699507490304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761408.763930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507490304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761408.763940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761408.763943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761408.763947:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507490304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761408.763954:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507490432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761408.763979:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761408.763992:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761408/real 1789761408] req@ffff8d25d25b9c00 x1876699507490432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761408.764026:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507490432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761408.764035:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761408.764040:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761408.764045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507490432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761408.764054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507490560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761408.764081:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761408.764097:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761408/real 1789761408] req@ffff8d25d25bb100 x1876699507490560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761408.764182:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507490560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761408.764191:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761408.764195:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761408.764198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507490560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761408.764205:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507490688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761408.764228:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761408.764240:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761408/real 1789761408] req@ffff8d25d25b8700 x1876699507490688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761408.764248:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507490688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761408.764254:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761408.764256:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761408.764258:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507490688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761408.764271:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507490816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761408.764285:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761408.764397:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761408/real 1789761408] req@ffff8d25d25bad80 x1876699507490816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761408.764450:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507490816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761433 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761408.764457:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761408.764461:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761408.764465:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507490816:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761408.827513:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761409.851609:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761410.875501:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761411.899514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761412.923523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761413.883481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761413.883494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761413.883500:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761413.883515:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1391 00000100:00080000:2.0:1789761413.883527:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761413.883536:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761413.883585:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507490944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761413.883616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761413.883621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761413.883623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761413.883630:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1391 00000100:00080000:2.0:1789761413.883634:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761413.883639:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761413.883652:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761413.883655:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507490944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761413.883670:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507491072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761413.883693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761413.883698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761413.883700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761413.883705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1391 00000100:00080000:2.0:1789761413.883717:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761413.883723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761413.883729:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761413.883730:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761413.883746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507491200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761413.883758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761413.883762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761413.883764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761413.883769:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1391 00000100:00080000:2.0:1789761413.883775:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761413.883776:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761413/real 1789761413] req@ffff8d25c8023480 x1876699507490944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761413.883783:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761413.883796:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507490944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761413.883798:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:3.0:1789761413.883811:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761413.883815:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761413.883820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507491328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761413.883821:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507490944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761413.883839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761413.883843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761413.883845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761413.883849:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761413.883852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1391 00000100:00080000:2.0:1789761413.883859:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761413.883864:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761413.883870:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761413.883882:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761413.883894:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507491456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761413.883896:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761413/real 1789761413] req@ffff8d25c8021880 x1876699507491072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761413.883930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507491072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761413.883931:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761413.883941:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761413.883946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761413.883968:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761413.883973:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1391 00000100:00080000:3.0:1789761413.883975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761413.883981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761413.883982:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761413.883989:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491200:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761413.883991:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761413.884000:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761413.884017:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507491584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761413.884028:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761413.884052:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761413/real 1789761413] req@ffff8d25c8021c00 x1876699507491200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761413.884067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507491200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761413.884076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761413.884079:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761413.884083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761413.884089:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761413.884112:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761413.884123:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761413/real 1789761413] req@ffff8d25c8021500 x1876699507491328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761413.884135:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507491328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761413.884142:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761413.884145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761413.884149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761413.884165:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507491456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761413.884186:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761413.884196:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761413/real 1789761413] req@ffff8d25c8020e00 x1876699507491456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761413.884204:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507491456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761413.884209:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761413.884211:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761413.884214:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507491456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761413.884217:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507491584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761413.884229:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761413.884235:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761413/real 1789761413] req@ffff8d25c8023480 x1876699507491584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761413.884243:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507491584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761438 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761413.884255:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761413.884257:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761413.884259:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507491584:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761413.947590:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761414.971532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761415.995545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761417.019510:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761418.043580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761419.003504:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761419.003515:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761419.003520:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761419.003532:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1396 00000100:00080000:2.0:1789761419.003543:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761419.003551:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761419.003594:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507491712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761419.003620:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761419.003623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761419.003624:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761419.003628:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1396 00000100:00080000:2.0:1789761419.003631:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761419.003635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761419.003646:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761419.003653:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507491712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761419.003662:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507491840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761419.003671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761419.003674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761419.003675:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761419.003678:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1396 00000100:00080000:2.0:1789761419.003680:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761419.003684:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761419.003688:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761419.003697:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507491968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761419.003704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761419.003707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761419.003708:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761419.003711:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1396 00000100:00080000:2.0:1789761419.003723:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761419.003725:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761419.003729:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761419.003738:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507492096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761419.003742:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761419.003752:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761419.003755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761419.003756:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761419.003760:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1396 00000100:00080000:2.0:1789761419.003763:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761419.003766:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761419.003769:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761419.003778:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507492224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761419.003784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761419.003788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761419.003790:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761419.003794:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761419] req@ffff8d25c8020000 x1876699507491712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761419.003795:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1396 00000100:00080000:2.0:1789761419.003801:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761419.003808:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761419.003814:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761419.003822:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507491712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761419.003825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507492352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761419.003838:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761419.003845:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761419.003850:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507491712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761419.003874:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761419.003890:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761419.003900:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761419] req@ffff8d25c8023100 x1876699507491840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761419.003908:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507491840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761419.003914:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761419.003933:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761419.003936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761419.003941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761419.003957:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761419.003965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761419] req@ffff8d25c8022300 x1876699507491968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761419.003988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507491968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761419.003997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761419.004001:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761419.004005:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507491968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761419.004012:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761419.004035:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761419.004047:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761419] req@ffff8d25c8022d80 x1876699507492096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761419.004061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507492096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761419.004069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761419.004072:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761419.004076:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761419.004082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507492224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761419.004105:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761419.004118:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761419] req@ffff8d25c8020700 x1876699507492224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761419.004130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507492224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761419.004145:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761419.004149:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761419.004152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507492224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761419.004159:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507492352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761419.004180:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761419.004193:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761419/real 1789761419] req@ffff8d25c8023b80 x1876699507492352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761419.004206:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507492352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761444 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761419.004214:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761419.004217:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761419.004221:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507492352:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761419.067580:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761420.091580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761421.115540:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761422.139554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761423.163545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761424.123473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.123484:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761424.123488:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.123498:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1401 00000100:00080000:0.0:1789761424.123508:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761424.123515:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761424.123547:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507492480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761424.123569:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.123572:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761424.123573:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.123577:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1401 00000100:00080000:0.0:1789761424.123579:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761424.123582:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761424.123589:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761424.123610:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507492608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761424.123618:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507492480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761424.123626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.123631:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761424.123633:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.123638:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1401 00000100:00080000:0.0:1789761424.123643:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761424.123648:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761424.123654:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761424.123668:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507492736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761424.123676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.123678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761424.123680:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.123682:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1401 00000100:00080000:0.0:1789761424.123686:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761424.123688:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761424.123692:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761424.123697:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761424.123706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507492864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761424.123714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.123716:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761424.123718:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761424.123720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1401 00000100:00080000:0.0:1789761424.123723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761424.123726:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761424.123729:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761424.123735:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffff8d25d25b9f80 x1876699507492480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761424.123739:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507492992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761424.123768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761424.123773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761424.123775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761424.123778:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507492480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761424.123782:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1401 00000100:00080000:0.0:1789761424.123791:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761424.123794:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761424.123798:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761424.123802:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761424.123807:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507492480:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761424.123808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761424.123826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507493120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761424.123834:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761424.123867:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761424.123893:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffff8d25d25bb800 x1876699507492608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761424.123909:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507492608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761424.123918:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761424.123922:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761424.123926:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761424.123933:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761424.123954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761424.123966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffff8d25d25b8380 x1876699507492736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761424.123979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507492736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761424.123988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761424.123991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761424.123995:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761424.124001:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761424.124017:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761424.124026:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffff8d25d25ba680 x1876699507492864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761424.124039:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507492864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761424.124047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761424.124050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761424.124053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507492864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761424.124059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507492992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761424.124074:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761424.124083:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffff8d25d25b9180 x1876699507492992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761424.124095:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507492992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761424.124104:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761424.124106:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761424.124110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507492992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761424.124127:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507493120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761424.124142:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761424.124150:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761424/real 1789761424] req@ffff8d25d25b8a80 x1876699507493120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761424.124163:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507493120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761449 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761424.124168:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761424.124170:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761424.124172:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507493120:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761424.187525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761425.211530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761426.235559:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761427.259557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761428.283519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761429.243450:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761429.243458:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761429.243462:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761429.243471:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1406 00000100:00080000:0.0:1789761429.243481:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761429.243487:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761429.243524:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507493248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761429.243544:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761429.243547:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761429.243548:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761429.243551:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1406 00000100:00080000:0.0:1789761429.243553:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761429.243556:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761429.243564:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761429.243573:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507493376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761429.243581:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507493248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761429.243584:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761429.243589:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761429.243591:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761429.243596:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1406 00000100:00080000:0.0:1789761429.243605:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761429.243611:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761429.243616:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761429.243631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507493504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761429.243637:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761429.243649:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761429.243654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761429.243656:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761429.243661:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1406 00000100:00080000:0.0:1789761429.243665:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761429.243670:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761429.243675:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761429.243675:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffff8d25d25bbb80 x1876699507493248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761429.243690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507493632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761429.243696:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507493248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761429.243706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761429.243713:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761429.243715:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761429.243716:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761429.243720:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1406 00000100:00080000:0.0:1789761429.243724:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761429.243724:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761429.243731:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761429.243733:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507493248:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761429.243736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761429.243747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507493760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761429.243753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761429.243755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761429.243757:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761429.243760:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507493376:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761429.243763:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1406 00000100:00080000:0.0:1789761429.243768:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761429.243770:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761429.243773:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761429.243781:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507493888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761429.243789:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761429.243810:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffff8d25d25b8000 x1876699507493376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761429.243824:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507493376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761429.243854:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761429.243859:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761429.243863:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507493376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761429.243870:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507493504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761429.243893:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761429.243904:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffff8d25d25ba300 x1876699507493504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761429.243913:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507493504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761429.243919:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761429.243921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761429.243923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507493504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761429.243927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507493632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761429.243941:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761429.243947:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffff8d25d25b8a80 x1876699507493632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761429.243955:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507493632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761429.243960:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761429.243962:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761429.243965:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507493632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761429.243969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507493760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761429.243979:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761429.243985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffff8d25d25b9180 x1876699507493760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761429.243992:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507493760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761429.243997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761429.243999:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761429.244001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507493760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761429.244012:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507493888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761429.244023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761429.244030:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761429/real 1789761429] req@ffff8d25d25ba680 x1876699507493888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761429.244037:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507493888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761454 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761429.244042:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761429.244044:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761429.244045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507493888:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761429.307551:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761430.331548:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761431.355580:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761432.379505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761433.403608:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761434.363516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761434.363528:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761434.363533:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.363542:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1412 00000100:00080000:2.0:1789761434.363552:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761434.363559:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761434.363596:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507494016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761434.363624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761434.363628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761434.363630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.363634:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1412 00000100:00080000:2.0:1789761434.363639:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761434.363644:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761434.363654:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761434.363667:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507494016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761434.363673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507494144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761434.363688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761434.363692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761434.363694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.363698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1412 00000100:00080000:2.0:1789761434.363713:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761434.363718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761434.363723:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761434.363725:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761434.363742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507494272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761434.363752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761434.363755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761434.363757:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.363762:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1412 00000100:00100000:3.0:1789761434.363766:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffff8d25c8020380 x1876699507494016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761434.363769:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761434.363779:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761434.363787:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761434.363788:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507494016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761434.363800:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761434.363820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507494400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761434.363822:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761434.363832:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507494016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761434.363852:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761434.363856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761434.363864:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761434.363868:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761434.363879:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1412 00000100:00080000:2.0:1789761434.363886:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761434.363892:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761434.363894:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761434.363902:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761434.363907:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffff8d25c8023800 x1876699507494144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761434.363921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507494528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761434.363922:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507494144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761434.363956:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761434.363961:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761434.363961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761434.363966:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494144:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761434.363969:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761434.363973:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761434.363974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761434.363985:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1412 00000100:00080000:2.0:1789761434.363992:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761434.363996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761434.364001:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761434.364009:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761434.364010:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffff8d25c8022a00 x1876699507494272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761434.364028:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507494656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761434.364032:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507494272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761434.364048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761434.364052:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761434.364057:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761434.364065:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761434.364087:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761434.364099:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffff8d25c8022680 x1876699507494400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761434.364107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507494400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761434.364114:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761434.364116:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761434.364118:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761434.364130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507494528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761434.364144:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761434.364151:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffff8d25c8021180 x1876699507494528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761434.364159:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507494528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761434.364169:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761434.364171:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761434.364174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507494528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761434.364177:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507494656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761434.364187:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761434.364192:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761434/real 1789761434] req@ffff8d25c8020a80 x1876699507494656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761434.364200:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507494656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761459 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761434.364205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761434.364207:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761434.364209:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507494656:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761434.427492:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761435.451537:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761436.475512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761437.499557:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761438.523553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761439.483454:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761439.483463:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761439.483468:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761439.483476:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1417 00000100:00080000:2.0:1789761439.483483:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761439.483489:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761439.483520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507494784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761439.483542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761439.483546:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761439.483548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761439.483553:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1417 00000100:00080000:2.0:1789761439.483557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761439.483563:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761439.483574:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761439.483580:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507494784:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761439.483593:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507494912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761439.483606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761439.483610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761439.483612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761439.483617:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1417 00000100:00080000:2.0:1789761439.483631:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761439.483636:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761439.483642:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761439.483644:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761439.483661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507495040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761439.483680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761439.483684:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761439/real 1789761439] req@ffff8d25c8020a80 x1876699507494784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761439.483686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761439.483691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761439.483701:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1417 00000100:00080000:2.0:1789761439.483706:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761439.483708:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507494784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761439.483714:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761439.483724:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:3.0:1789761439.483729:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761439.483737:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761439.483742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507495168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761439.483747:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507494784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761439.483763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761439.483778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761439.483780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761439.483784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1417 00000100:00080000:2.0:1789761439.483788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761439.483795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761439.483796:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494912:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789761439.483799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761439.483817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507495296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761439.483826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761439.483831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:3.0:1789761439.483832:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761439.483833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761439.483843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1417 00000100:00080000:2.0:1789761439.483847:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761439.483849:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761439.483855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761439/real 1789761439] req@ffff8d25c8021180 x1876699507494912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761439.483857:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761439.483875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507495424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761439.483877:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507494912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761439.483907:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761439.483912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761439.483917:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507494912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761439.483923:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761439.483940:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761439.483949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761439/real 1789761439] req@ffff8d25c8022680 x1876699507495040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761439.483961:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507495040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761439.483970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761439.483973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761439.483976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761439.483983:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761439.484006:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761439.484020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761439/real 1789761439] req@ffff8d25c8022a00 x1876699507495168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761439.484034:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507495168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761439.484043:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761439.484047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761439.484051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761439.484068:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507495296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761439.484087:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761439.484095:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761439/real 1789761439] req@ffff8d25c8023800 x1876699507495296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761439.484103:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507495296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761439.484109:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761439.484111:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761439.484113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507495296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761439.484117:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507495424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761439.484127:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761439.484133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761439/real 1789761439] req@ffff8d25c8020380 x1876699507495424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761439.484145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507495424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761464 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761439.484151:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761439.484152:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761439.484154:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507495424:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761439.547516:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761440.571569:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761441.595555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761442.619545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761443.643518:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761444.603516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761444.603537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761444.603546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761444.603557:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1422 00000100:00080000:2.0:1789761444.603571:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761444.603580:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761444.603621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507495552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761444.603651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761444.603655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761444.603657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761444.603661:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1422 00000100:00080000:2.0:1789761444.603664:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761444.603669:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761444.603679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761444.603686:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507495552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761444.603700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507495680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761444.603717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761444.603720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761444.603723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761444.603740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1422 00000100:00080000:2.0:1789761444.603745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761444.603750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761444.603755:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761444.603766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507495808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761444.603775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761444.603781:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761444.603783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761444.603787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761444.603795:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1422 00000100:00080000:2.0:1789761444.603801:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761444.603806:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761444.603813:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761444.603830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507495936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761444.603831:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761444/real 1789761444] req@ffff8d25c8023b80 x1876699507495552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761444.603864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761444.603869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761444.603871:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761444.603876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1422 00000100:00100000:0.0:1789761444.603878:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507495552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761444.603882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761444.603891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761444.603894:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789761444.603896:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761444.603900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761444.603907:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507495552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761444.603912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507496064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761444.603921:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761444.603924:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761444.603925:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761444.603928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1422 00000100:00080000:2.0:1789761444.603930:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761444.603933:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761444.603937:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761444.603941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761444.603947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507496192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761444.603967:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761444.603981:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761444/real 1789761444] req@ffff8d25c8020700 x1876699507495680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761444.603990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507495680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761444.603997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761444.604000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761444.604002:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761444.604006:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761444.604033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761444.604041:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761444/real 1789761444] req@ffff8d25c8022d80 x1876699507495808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761444.604050:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507495808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761444.604061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761444.604064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761444.604066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761444.604071:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761444.604084:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761444.604092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761444/real 1789761444] req@ffff8d25c8022300 x1876699507495936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761444.604100:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507495936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761444.604106:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761444.604109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761444.604112:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507495936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761444.604118:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761444.604140:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761444.604157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761444/real 1789761444] req@ffff8d25c8023100 x1876699507496064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761444.604176:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507496064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761444.604183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761444.604186:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761444.604190:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761444.604207:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761444.604221:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761444.604229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761444/real 1789761444] req@ffff8d25c8020000 x1876699507496192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761444.604245:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507496192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761469 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761444.604250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761444.604253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761444.604255:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496192:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761444.667541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761445.691556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761446.715635:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761447.739530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761448.763543:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761449.723471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761449.723480:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761449.723484:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761449.723495:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1427 00000100:00080000:3.0:1789761449.723504:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761449.723512:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761449.723550:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507496320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761449.723581:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761449.723585:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761449.723586:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761449.723592:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1427 00000100:00080000:3.0:1789761449.723595:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761449.723599:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761449.723607:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761449.723619:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507496448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761449.723626:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507496320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761449.723631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761449.723635:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761449.723636:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761449.723640:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1427 00000100:00080000:3.0:1789761449.723642:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761449.723645:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761449.723648:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761449.723657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507496576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761449.723664:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761449.723667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761449.723668:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761449.723670:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1427 00000100:00080000:3.0:1789761449.723674:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761449.723675:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761449.723678:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761449.723701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507496704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761449.723705:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761449.723714:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761449.723718:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761449.723720:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761449.723724:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1427 00000100:00080000:3.0:1789761449.723728:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761449.723731:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761449.723736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761449.723744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507496832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761449.723751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761449.723754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761449.723755:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761449.723760:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1427 00000100:00100000:0.0:1789761449.723762:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761449/real 1789761449] req@ffff8d25d1b0bb80 x1876699507496320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761449.723762:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761449.723767:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761449.723771:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761449.723782:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507496960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761449.723795:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507496320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761449.723814:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761449.723817:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761449.723821:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507496320:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761449.723840:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507496448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761449.723859:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761449.723869:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761449/real 1789761449] req@ffff8d25d1b08700 x1876699507496448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761449.723878:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507496448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761449.723883:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761449.723886:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761449.723888:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507496448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761449.723892:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507496576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761449.723902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761449.723908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761449/real 1789761449] req@ffff8d25d1b0b800 x1876699507496576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761449.723916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507496576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761449.723921:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761449.723923:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761449.723925:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507496576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761449.723928:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507496704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761449.723939:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761449.723944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761449/real 1789761449] req@ffff8d25d1b08e00 x1876699507496704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761449.723957:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507496704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761449.723966:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761449.723970:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761449.723973:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507496704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761449.723979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761449.724000:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761449.724012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761449/real 1789761449] req@ffff8d25d1b0a680 x1876699507496832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761449.724025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507496832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761449.724031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761449.724033:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761449.724036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761449.724040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761449.724054:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761449.724061:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761449/real 1789761449] req@ffff8d25d1b09880 x1876699507496960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761449.724069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507496960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761474 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761449.724078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761449.724080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761449.724082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507496960:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761449.787503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761450.811547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761451.835509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761452.859520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761453.883512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761454.843493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761454.843504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761454.843509:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761454.843519:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1432 00000100:00080000:3.0:1789761454.843531:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761454.843538:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761454.843586:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507497088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761454.843626:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761454.843631:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761454.843633:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761454.843638:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1432 00000100:00080000:3.0:1789761454.843643:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761454.843648:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761454.843676:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761454.843688:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507497088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761454.843697:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507497216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761454.843719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761454.843724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761454.843726:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761454.843731:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1432 00000100:00080000:3.0:1789761454.843736:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761454.843740:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761454.843744:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761454.843756:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507497344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761454.843760:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761454.843768:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761454.843771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761454.843773:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761454.843777:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1432 00000100:00080000:3.0:1789761454.843780:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761454.843783:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761454.843786:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761454.843797:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507497472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761454.843809:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761454.843813:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761454.843816:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761454.843817:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761454/real 1789761454] req@ffff8d25d1b09880 x1876699507497088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761454.843825:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1432 00000100:00080000:3.0:1789761454.843833:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761454.843839:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761454.843842:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507497088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761454.843847:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761454.843854:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761454.843858:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761454.843865:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507497088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761454.843865:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507497600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761454.843878:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761454.843882:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761454.843884:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761454.843889:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1432 00000100:00100000:0.0:1789761454.843894:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497216:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761454.843894:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761454.843903:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761454.843911:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761454.843918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761454.843930:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507497728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761454.843931:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761454/real 1789761454] req@ffff8d25d1b0a680 x1876699507497216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761454.843964:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507497216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761454.843985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761454.843989:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761454.843994:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761454.843999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761454.844013:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761454.844020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761454/real 1789761454] req@ffff8d25d1b08e00 x1876699507497344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761454.844029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507497344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761454.844035:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761454.844038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761454.844040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761454.844043:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761454.844054:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761454.844060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761454/real 1789761454] req@ffff8d25d1b0b800 x1876699507497472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761454.844073:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507497472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761454.844079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761454.844082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761454.844084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761454.844087:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507497600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761454.844099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761454.844107:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761454/real 1789761454] req@ffff8d25d1b08700 x1876699507497600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761454.844120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507497600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761454.844129:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761454.844133:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761454.844137:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507497600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761454.844161:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507497728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761454.844185:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761454.844199:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761454/real 1789761454] req@ffff8d25d1b0bb80 x1876699507497728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761454.844211:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507497728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761479 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761454.844220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761454.844224:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761454.844228:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507497728:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761454.907551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761455.931473:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761456.955558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761457.979694:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761459.003544:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761459.963531:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761459.963545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761459.963550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761459.963562:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1437 00000100:00080000:2.0:1789761459.963579:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761459.963586:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761459.963641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507497856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761459.963666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761459.963670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761459.963673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761459.963678:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1437 00000100:00080000:2.0:1789761459.963682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761459.963688:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761459.963700:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761459.963716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507497984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761459.963723:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507497856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761459.963728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761459.963732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761459.963733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761459.963737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1437 00000100:00080000:2.0:1789761459.963740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761459.963743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761459.963748:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761459.963759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507498112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761459.963766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761459.963769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761459.963771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761459.963774:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1437 00000100:00080000:2.0:1789761459.963777:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761459.963780:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761459.963784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761459.963791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507498240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761459.963798:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761459.963806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761459.963809:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761459.963810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761459.963814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1437 00000100:00080000:2.0:1789761459.963817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761459.963821:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761459.963824:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761459.963833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507498368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761459.963839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761459.963842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761459.963844:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761459.963848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761459/real 1789761459] req@ffff8d25c8023480 x1876699507497856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761459.963848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1437 00000100:00080000:2.0:1789761459.963850:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761459.963857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761459.963864:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761459.963877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507498496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761459.963885:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507497856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761459.963899:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761459.963910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761459.963914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507497856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761459.963937:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761459.963954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761459.963967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761459/real 1789761459] req@ffff8d25c8020e00 x1876699507497984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761459.963980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507497984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761459.963989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761459.963992:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761459.963996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507497984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761459.964003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761459.964025:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761459.964039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761459/real 1789761459] req@ffff8d25c8021500 x1876699507498112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761459.964053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507498112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761459.964067:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761459.964069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761459.964072:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761459.964077:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761459.964102:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761459.964111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761459/real 1789761459] req@ffff8d25c8021c00 x1876699507498240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761459.964119:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507498240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761459.964125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761459.964127:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761459.964129:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761459.964132:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507498368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761459.964149:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761459.964156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761459/real 1789761459] req@ffff8d25c8021880 x1876699507498368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761459.964163:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507498368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761459.964169:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761459.964171:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761459.964174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507498368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761459.964178:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507498496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761459.964189:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761459.964195:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761459/real 1789761459] req@ffff8d25c8020000 x1876699507498496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761459.964210:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507498496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761484 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761459.964219:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761459.964222:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761459.964226:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507498496:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761460.027557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761461.051538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761462.075547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761463.099531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761464.123560:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761465.083464:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761465.083475:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761465.083479:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761465.083492:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1442 00000100:00080000:2.0:1789761465.083503:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761465.083510:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761465.083552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507498624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761465.083600:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761465.083604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761465.083605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761465.083609:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1442 00000100:00080000:2.0:1789761465.083612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761465.083616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761465.083626:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761465.083638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507498752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761465.083646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761465.083651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761465.083652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507498624:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761465.083653:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761465.083662:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1442 00000100:00080000:2.0:1789761465.083665:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761465.083668:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761465.083671:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761465.083681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507498880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761465.083687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761465.083690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761465.083691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761465.083694:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1442 00000100:00080000:2.0:1789761465.083697:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761465.083700:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761465.083703:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761465.083710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507499008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761465.083717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761465.083720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761465.083721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761465.083724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1442 00000100:00080000:2.0:1789761465.083726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761465.083729:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761465.083732:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761465.083738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507499136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761465.083745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761465.083748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761465.083752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761465.083757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1442 00000100:00080000:2.0:1789761465.083762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761465.083765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761465.083769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761465.083774:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761465.083780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507499264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761465.083814:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761465.083833:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761465.083848:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507499136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761465.083868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507499264:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761465.147547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761466.171517:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761467.195478:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761468.155641:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761468.155675:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761468.155687:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761468.155725:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761468.155735:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761468.155742:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761468.155751:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761468.155761:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761468.155772:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761468] req@ffff8d25c8022300 x1876699507498752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761468.155798:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507498752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761468.155811:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761468.155818:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761468.155822:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761468.155841:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761468] req@ffff8d25c8022d80 x1876699507498880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761468.155854:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507498880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761468.155863:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761468.155867:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761468.155870:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507498880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761468.155877:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761468] req@ffff8d25c8020700 x1876699507499008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761468.155889:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507499008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761468.155897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761468.155900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761468.155903:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761468.155910:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761468] req@ffff8d25c8023b80 x1876699507499136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761468.155921:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507499136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761468.155929:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761468.155932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761468.155936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507499136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761468.155942:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761468] req@ffff8d25c8023100 x1876699507498624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761468.155955:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507498624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761468.155962:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761468.155966:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761468.155969:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507498624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761468.156000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761465/real 1789761468] req@ffff8d25c8020380 x1876699507499264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761468.156012:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507499264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761490 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761468.156020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761468.156023:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761468.156027:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507499264:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761468.219565:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761469.243514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761470.203485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761470.203499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761470.203505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761470.203520:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1447 00000100:00080000:2.0:1789761470.203552:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761470.203561:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761470.203602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507499392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761470.203628:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761470.203632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761470.203633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761470.203637:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1447 00000100:00080000:2.0:1789761470.203641:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761470.203645:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761470.203657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761470.203671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507499520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761470.203683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761470.203685:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507499392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761470.203687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761470.203690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761470.203705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1447 00000100:00080000:2.0:1789761470.203710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761470.203715:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761470.203722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761470.203742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507499648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761470.203755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761470.203763:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761470.203770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761470.203774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761470.203783:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1447 00000100:00080000:2.0:1789761470.203789:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761470.203796:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761470.203804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761470.203828:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffff8d25c8023800 x1876699507499392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761470.203841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507499776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761470.203859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507499392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761470.203866:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761470.203884:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761470.203890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761470.203895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761470.203899:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1447 00000100:00080000:0.0:1789761470.203906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761470.203911:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761470.203913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507499392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761470.203922:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761470.203938:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761470.203955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761470.203960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507499904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761470.203980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761470.203985:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761470.203988:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761470.204000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1447 00000800:00000100:0.0:1789761470.204002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761470.204012:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761470.204021:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761470.204030:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761470.204031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffff8d25c8022a00 x1876699507499520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761470.204057:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507500032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761470.204058:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507499520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.204130:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.204135:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761470.204140:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761470.204150:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761470.204180:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761470.204196:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffff8d25c8022680 x1876699507499648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761470.204223:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507499648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.204233:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.204236:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761470.204240:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761470.204248:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761470.204272:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761470.204284:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffff8d25c8021180 x1876699507499776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761470.204296:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507499776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.204304:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.204308:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761470.204311:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507499776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761470.204338:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507499904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761470.204363:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761470.204410:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffff8d25c8020a80 x1876699507499904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761470.204425:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507499904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.204439:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.204442:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761470.204445:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507499904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761470.204451:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507500032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761470.204475:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761470.204488:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761470/real 1789761470] req@ffff8d25c8023800 x1876699507500032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761470.204500:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507500032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761495 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761470.204508:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761470.204512:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761470.204517:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507500032:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761470.267553:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761471.291495:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761472.315556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761473.339548:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761474.363617:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761475.323517:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761475.323531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761475.323537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761475.323551:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1453 00000100:00080000:2.0:1789761475.323564:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761475.323573:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761475.323622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507500160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761475.323653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761475.323657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761475.323659:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761475.323666:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1453 00000100:00080000:2.0:1789761475.323670:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761475.323675:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761475.323684:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507500160:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761475.323687:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761475.323704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507500288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761475.323716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761475.323721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761475.323723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761475.323728:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1453 00000100:00080000:2.0:1789761475.323732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761475.323737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761475.323743:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761475.323756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507500416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761475.323758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761475.323771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761475.323776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761475.323779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761475.323785:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1453 00000100:00080000:2.0:1789761475.323798:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761475.323803:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761475.323810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761475.323811:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffff8d25c8020380 x1876699507500160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761475.323830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507500544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761475.323832:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507500160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761475.323871:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761475.323874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761475.323877:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761475.323883:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789761475.323884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507500160:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761475.323895:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761475.323903:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1453 00000100:00080000:2.0:1789761475.323908:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761475.323912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507500288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761475.323918:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761475.323926:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761475.323936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761475.323943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507500672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761475.323949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffff8d25c8023100 x1876699507500288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761475.323973:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761475.323976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507500288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761475.323986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761475.323990:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761475.323991:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761475.324000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1453 00000100:00080000:2.0:1789761475.324004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761475.324007:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761475.324011:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761475.324018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507500288:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789761475.324020:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761475.324029:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507500416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761475.324037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507500800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761475.324075:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761475.324090:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffff8d25c8023b80 x1876699507500416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761475.324100:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507500416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761475.324105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761475.324107:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761475.324110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507500416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761475.324114:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507500544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761475.324130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761475.324137:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffff8d25c8020700 x1876699507500544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761475.324145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507500544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761475.324150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761475.324152:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761475.324154:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507500544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761475.324166:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507500672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761475.324178:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761475.324184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffff8d25c8022d80 x1876699507500672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761475.324192:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507500672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761475.324198:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761475.324200:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761475.324202:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507500672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761475.324205:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507500800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761475.324214:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761475.324220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761475/real 1789761475] req@ffff8d25c8022300 x1876699507500800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761475.324232:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507500800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761500 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761475.324237:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761475.324239:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761475.324240:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507500800:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761475.387521:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761476.411572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761477.435570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761478.459546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761479.483556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761480.443586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761480.443594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761480.443599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761480.443612:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1458 00000100:00080000:2.0:1789761480.443623:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761480.443631:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761480.443671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507500928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761480.443696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761480.443700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761480.443702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761480.443710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1458 00000100:00080000:2.0:1789761480.443718:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761480.443724:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761480.443735:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761480.443748:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507500928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761480.443750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507501056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761480.443776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761480.443782:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761480.443784:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761480.443790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1458 00000100:00080000:2.0:1789761480.443807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761480.443814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761480.443819:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761480.443830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507501184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761480.443837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761480.443840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761480.443841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761480.443845:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1458 00000100:00080000:2.0:1789761480.443848:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761480.443851:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761480.443856:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761480.443863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507501312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761480.443871:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761480.443872:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761480.443880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761480.443882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761480.443888:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1458 00000100:00080000:2.0:1789761480.443893:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761480.443897:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761480.443902:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761480.443910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507501440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761480.443916:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761480.443920:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761480.443921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761480.443928:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1458 00000100:00080000:2.0:1789761480.443932:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761480.443936:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761480.443941:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761480.443953:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507501568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761480.444064:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761480/real 1789761480] req@ffff8d25c8020000 x1876699507500928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761480.444101:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507500928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761480.444114:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761480.444123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761480.444133:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507500928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761480.444154:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761480.444175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761480.444185:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761480/real 1789761480] req@ffff8d25c8021880 x1876699507501056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761480.444194:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507501056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761480.444204:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761480.444207:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761480.444209:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761480.444213:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761480.444224:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761480.444231:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761480/real 1789761480] req@ffff8d25c8021c00 x1876699507501184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761480.444239:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507501184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761480.444244:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761480.444246:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761480.444248:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761480.444252:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761480.444263:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761480.444269:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761480/real 1789761480] req@ffff8d25c8021500 x1876699507501312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761480.444276:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507501312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761480.444282:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761480.444283:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761480.444286:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761480.444290:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507501440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761480.444301:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761480.444307:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761480/real 1789761480] req@ffff8d25c8020e00 x1876699507501440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761480.444314:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507501440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761480.444319:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761480.444321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761480.444323:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507501440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761480.444326:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507501568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761480.444337:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761480.444343:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761480/real 1789761480] req@ffff8d25c8023480 x1876699507501568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761480.444351:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507501568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761505 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761480.444356:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761480.444358:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761480.444359:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507501568:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761480.507572:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761481.531570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761482.555548:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761483.579556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761484.603522:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761485.563497:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761485.563510:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761485.563516:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761485.563528:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1463 00000100:00080000:2.0:1789761485.563540:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761485.563548:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761485.563600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507501696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761485.563635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761485.563638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761485.563640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761485.563644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1463 00000100:00080000:2.0:1789761485.563647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761485.563651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761485.563662:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761485.563674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507501824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761485.563681:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507501696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761485.563685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761485.563689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761485.563691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761485.563695:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1463 00000100:00080000:2.0:1789761485.563697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761485.563699:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761485.563702:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761485.563710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507501952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761485.563717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761485.563720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761485.563721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761485.563723:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1463 00000100:00080000:2.0:1789761485.563725:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761485.563728:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761485.563731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761485.563739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507502080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761485.563746:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761485.563749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761485.563751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761485.563754:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1463 00000100:00080000:2.0:1789761485.563757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761485.563760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761485.563762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761485.563771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507502208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761485.563774:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761485.563782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761485.563785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761485.563786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761485.563790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1463 00000100:00080000:2.0:1789761485.563792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761485.563795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761485.563798:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761485.563805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507502336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761485.563833:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761485/real 1789761485] req@ffff8d25c8023480 x1876699507501696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761485.563850:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507501696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761485.563859:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761485.563863:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761485.563866:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507501696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761485.563888:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761485.563908:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761485.563917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761485/real 1789761485] req@ffff8d25c8020e00 x1876699507501824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761485.563925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507501824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761485.563931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761485.563933:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761485.563935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761485.563939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761485.563950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761485.563956:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761485/real 1789761485] req@ffff8d25c8021500 x1876699507501952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761485.563976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507501952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761485.563985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761485.563988:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761485.563991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507501952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761485.563998:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761485.564021:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761485.564034:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761485/real 1789761485] req@ffff8d25c8021c00 x1876699507502080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761485.564047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507502080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761485.564060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761485.564064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761485.564068:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761485.564075:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507502208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761485.564097:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761485.564109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761485/real 1789761485] req@ffff8d25c8021880 x1876699507502208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761485.564122:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507502208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761485.564136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761485.564140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761485.564143:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507502208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761485.564149:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507502336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761485.564163:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761485.564171:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761485/real 1789761485] req@ffff8d25c8020000 x1876699507502336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761485.564180:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507502336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761510 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761485.564188:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761485.564191:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761485.564195:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507502336:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761485.627614:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761486.651550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761487.675576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761488.699521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761489.723530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761490.683479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761490.683491:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761490.683495:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761490.683506:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1468 00000100:00080000:2.0:1789761490.683517:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761490.683525:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761490.683567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507502464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761490.683592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761490.683596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761490.683597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761490.683601:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1468 00000100:00080000:2.0:1789761490.683604:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761490.683608:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761490.683624:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761490.683625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507502464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761490.683637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507502592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761490.683645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761490.683649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761490.683650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761490.683654:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1468 00000100:00080000:2.0:1789761490.683656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761490.683659:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761490.683662:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761490.683673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507502720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761490.683680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761490.683683:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761490.683684:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761490.683688:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1468 00000100:00080000:2.0:1789761490.683691:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761490.683693:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761490.683698:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761490.683699:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761490.683708:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507502848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761490.683716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761490.683719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761490.683721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761490.683723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1468 00000100:00080000:2.0:1789761490.683726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761490.683729:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761490.683732:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761490.683739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507502976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761490.683745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761490.683748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761490.683749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761490.683753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1468 00000100:00080000:2.0:1789761490.683755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761490.683758:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761490/real 1789761490] req@ffff8d25c8022300 x1876699507502464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761490.683759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761490.683763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761490.683779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507503104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761490.683784:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507502464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761490.683803:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761490.683806:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761490.683812:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507502464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761490.683835:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761490.683854:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761490.683864:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761490/real 1789761490] req@ffff8d25c8022d80 x1876699507502592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761490.683873:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507502592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761490.683879:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761490.683882:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761490.683884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761490.683888:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761490.683900:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761490.683906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761490/real 1789761490] req@ffff8d25c8020700 x1876699507502720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761490.683914:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507502720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761490.683919:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761490.683921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761490.683923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761490.683926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761490.683937:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761490.683943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761490/real 1789761490] req@ffff8d25c8023b80 x1876699507502848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761490.683951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507502848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761490.683956:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761490.683958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761490.683960:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507502848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761490.683963:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507502976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761490.683973:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761490.683980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761490/real 1789761490] req@ffff8d25c8023100 x1876699507502976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761490.683987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507502976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761490.683992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761490.683994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761490.683996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507502976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761490.683999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507503104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761490.684017:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761490.684027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761490/real 1789761490] req@ffff8d25c8020380 x1876699507503104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761490.684035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507503104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761515 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761490.684040:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761490.684042:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761490.684045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507503104:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761490.747512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761491.771584:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761492.795525:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761493.819546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761494.843511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761495.803472:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761495.803484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761495.803489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761495.803500:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1473 00000100:00080000:2.0:1789761495.803510:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761495.803518:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761495.803559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507503232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761495.803587:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761495.803591:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761495.803593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761495.803599:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1473 00000100:00080000:2.0:1789761495.803603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761495.803608:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761495.803621:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761495.803632:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507503232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761495.803638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507503360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761495.803648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761495.803650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761495.803652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761495.803655:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1473 00000100:00080000:2.0:1789761495.803658:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761495.803662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761495.803666:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761495.803674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507503488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761495.803681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761495.803684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761495.803685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761495.803688:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1473 00000100:00080000:2.0:1789761495.803702:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761495.803705:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761495.803708:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761495.803715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507503616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761495.803724:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761495.803730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761495.803733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761495.803735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761495.803739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1473 00000100:00080000:2.0:1789761495.803742:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761495.803745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761495.803748:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761495.803755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507503744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761495.803761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761495.803765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761495.803766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761495.803769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1473 00000100:00080000:2.0:1789761495.803772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761495.803775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761495.803778:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761495.803783:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761495/real 1789761495] req@ffff8d25c8023800 x1876699507503232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761495.803788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507503872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761495.803828:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507503232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761495.803841:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761495.803846:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761495.803850:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507503232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761495.803879:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507503360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761495.803906:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761495.803921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761495/real 1789761495] req@ffff8d25c8020a80 x1876699507503360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761495.803935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507503360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761495.803943:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761495.803948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761495.803951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507503360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761495.803958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507503488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761495.803979:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761495.803991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761495/real 1789761495] req@ffff8d25c8021180 x1876699507503488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761495.804003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507503488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761495.804011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761495.804015:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761495.804018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507503488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761495.804025:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507503616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761495.804045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761495.804056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761495/real 1789761495] req@ffff8d25c8022680 x1876699507503616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761495.804069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507503616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761495.804079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761495.804082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761495.804086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507503616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761495.804092:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507503744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761495.804112:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761495.804124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761495/real 1789761495] req@ffff8d25c8022a00 x1876699507503744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761495.804136:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507503744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761495.804144:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761495.804148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761495.804151:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507503744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761495.804158:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507503872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761495.804179:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761495.804190:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761495/real 1789761495] req@ffff8d25c8020380 x1876699507503872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761495.804203:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507503872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761520 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761495.804211:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761495.804214:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761495.804218:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507503872:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761495.867555:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761496.891485:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761497.915535:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761498.939556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761499.963511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761500.923482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761500.923493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761500.923498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761500.923509:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1478 00000100:00080000:2.0:1789761500.923520:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761500.923529:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761500.923567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507504000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761500.923590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761500.923592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761500.923594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761500.923597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1478 00000100:00080000:2.0:1789761500.923601:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761500.923605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761500.923613:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761500.923620:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507504000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761500.923627:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507504128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761500.923636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761500.923638:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761500.923640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761500.923642:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1478 00000100:00080000:2.0:1789761500.923645:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761500.923647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761500.923651:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761500.923659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507504256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761500.923665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761500.923668:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761500.923669:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761500.923671:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1478 00000100:00080000:2.0:1789761500.923674:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761500.923677:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761500.923680:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761500.923685:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761500.923694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507504384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761500.923704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761500.923708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761500.923710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761500.923715:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761500/real 1789761500] req@ffff8d25c8023100 x1876699507504000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761500.923716:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1478 00000100:00080000:2.0:1789761500.923724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761500.923733:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761500.923739:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507504000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761500.923741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761500.923752:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761500.923756:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761500.923759:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507504000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761500.923764:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507504512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761500.923776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761500.923780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761500.923781:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504128:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761500.923783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761500.923792:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1478 00000100:00080000:2.0:1789761500.923798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761500.923802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761500.923807:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761500.923811:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761500.923820:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761500/real 1789761500] req@ffff8d25c8023b80 x1876699507504128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761500.923825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507504640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761500.923841:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507504128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761500.923849:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761500.923852:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761500.923855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761500.923860:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761500.923872:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761500.923879:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761500/real 1789761500] req@ffff8d25c8020700 x1876699507504256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761500.923888:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507504256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761500.923905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761500.923908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761500.923910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761500.923914:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761500.923926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761500.923933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761500/real 1789761500] req@ffff8d25c8022d80 x1876699507504384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761500.923940:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507504384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761500.923945:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761500.923947:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761500.923949:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761500.923953:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507504512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761500.923963:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761500.923968:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761500/real 1789761500] req@ffff8d25c8022300 x1876699507504512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761500.923976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507504512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761500.923980:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761500.923982:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761500.923985:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507504512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761500.923995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507504640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761500.924005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761500.924012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761500/real 1789761500] req@ffff8d25c8020000 x1876699507504640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761500.924019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507504640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761525 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761500.924025:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761500.924028:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761500.924031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507504640:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761500.987568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761502.011529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761503.035533:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761504.059517:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761505.083541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761506.043478:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761506.043486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761506.043490:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761506.043500:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1483 00000100:00080000:3.0:1789761506.043508:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761506.043514:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761506.043544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507504768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761506.043566:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761506.043569:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761506.043570:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761506.043573:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1483 00000100:00080000:3.0:1789761506.043577:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761506.043581:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761506.043589:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761506.043595:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507504768:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761506.043601:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507504896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761506.043610:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761506.043613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761506.043614:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761506.043617:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1483 00000100:00080000:3.0:1789761506.043619:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761506.043621:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761506.043625:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761506.043633:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507505024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761506.043644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761506.043647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761506.043649:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761506.043651:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761506.043655:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1483 00000100:00080000:3.0:1789761506.043659:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761506.043662:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761506.043666:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761506.043673:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507505152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761506.043685:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761506.043688:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761506.043689:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761506.043692:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1483 00000100:00080000:3.0:1789761506.043695:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761506.043697:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761506.043701:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761506.043704:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffff8d25d1b08a80 x1876699507504768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761506.043711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507505280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761506.043726:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761506.043729:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761506.043730:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761506.043734:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507504768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761506.043736:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1483 00000100:00080000:3.0:1789761506.043741:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761506.043747:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761506.043750:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761506.043751:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761506.043757:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761506.043761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507505408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761506.043763:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507504768:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761506.043791:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761506.043812:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761506.043822:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffff8d25d1b0aa00 x1876699507504896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.043835:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507504896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.043844:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.043847:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761506.043851:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507504896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761506.043857:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761506.043878:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761506.043888:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffff8d25d1b09180 x1876699507505024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.043896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507505024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.043902:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.043904:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761506.043906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761506.043909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761506.043922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761506.043928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffff8d25d1b0ad80 x1876699507505152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.043936:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507505152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.043942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.043943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761506.043945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761506.043949:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507505280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761506.043959:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761506.043965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffff8d25d1b09c00 x1876699507505280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.043974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507505280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.043982:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.043986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761506.043989:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507505280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761506.043995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507505408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761506.044015:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761506.044027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761506/real 1789761506] req@ffff8d25d1b0b100 x1876699507505408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761506.044040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507505408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761531 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761506.044055:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761506.044059:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761506.044062:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507505408:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761506.107499:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761507.131555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761508.155557:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761509.179540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761510.203493:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761511.163476:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761511.163487:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761511.163493:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761511.163504:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1488 00000100:00080000:3.0:1789761511.163514:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761511.163521:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761511.163553:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507505536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761511.163574:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761511.163577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761511.163578:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761511.163582:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1488 00000100:00080000:3.0:1789761511.163584:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761511.163587:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761511.163611:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761511.163620:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507505664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761511.163629:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761511.163633:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761511.163634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507505536:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761511.163634:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761511.163639:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1488 00000100:00080000:3.0:1789761511.163644:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761511.163648:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761511.163651:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761511.163658:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507505792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761511.163670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761511.163672:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761511.163674:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761511.163676:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1488 00000100:00080000:3.0:1789761511.163678:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761511.163681:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761511.163684:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761511.163690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507505920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761511.163697:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761511.163701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1789761511.163703:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761511.163704:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761511.163711:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1488 00000100:00080000:3.0:1789761511.163714:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761511.163716:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761511.163720:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761511.163727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507506048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761511.163733:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761511.163736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761511.163737:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761511.163740:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1488 00000100:00080000:3.0:1789761511.163744:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761511.163747:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffff8d25d1b0a300 x1876699507505536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761511.163748:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761511.163755:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761511.163769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507506176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761511.163772:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507505536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761511.163796:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761511.163800:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761511.163804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507505536:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761511.163825:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761511.163842:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761511.163852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffff8d25d1b08380 x1876699507505664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761511.163860:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507505664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761511.163866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761511.163868:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761511.163871:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761511.163875:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761511.163885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761511.163892:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffff8d25d1b08000 x1876699507505792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761511.163900:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507505792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761511.163905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761511.163906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761511.163908:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761511.163912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761511.163922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761511.163928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffff8d25d1b09500 x1876699507505920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761511.163935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507505920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761511.163940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761511.163942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761511.163944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507505920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761511.163947:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761511.163957:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761511.163963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffff8d25d1b0b480 x1876699507506048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761511.163971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507506048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761511.163977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761511.163978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761511.163980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761511.163983:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761511.163992:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761511.163998:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761511/real 1789761511] req@ffff8d25d1b0b100 x1876699507506176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761511.164006:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507506176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761536 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761511.164011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761511.164013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761511.164014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506176:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761511.227554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761512.251506:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761513.275538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761514.299574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761515.323523:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761516.283572:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761516.283578:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761516.283582:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761516.283591:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1493 00000100:00080000:0.0:1789761516.283602:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761516.283608:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761516.283639:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507506304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761516.283659:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761516.283662:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761516.283663:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761516.283667:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1493 00000100:00080000:0.0:1789761516.283670:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761516.283674:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761516.283682:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761516.283691:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507506304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761516.283696:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507506432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761516.283706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761516.283708:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761516.283710:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761516.283712:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1493 00000100:00080000:0.0:1789761516.283715:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761516.283719:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761516.283723:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761516.283738:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507506560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761516.283745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761516.283749:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761516.283753:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761516.283758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761516.283762:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1493 00000100:00080000:0.0:1789761516.283766:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761516.283769:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761516.283774:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761516.283783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507506688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761516.283798:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffff8d25d25b8380 x1876699507506304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761516.283799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761516.283807:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761516.283810:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761516.283814:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1493 00000100:00080000:0.0:1789761516.283817:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761516.283819:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761516.283822:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761516.283830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507506816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761516.283835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761516.283838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761516.283839:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761516.283841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1493 00000100:00080000:0.0:1789761516.283842:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761516.283844:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761516.283848:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761516.283854:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507506944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761516.283882:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507506304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761516.283891:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761516.283894:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761516.283897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507506304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761516.283915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507506432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761516.283933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761516.283942:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffff8d25d25bb800 x1876699507506432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761516.284063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507506432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761516.284074:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761516.284106:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761516.284110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507506432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761516.284116:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507506560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761516.284133:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761516.284144:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffff8d25d25b9f80 x1876699507506560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761516.284157:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507506560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761516.284164:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761516.284167:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761516.284174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507506560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761516.284180:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507506688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761516.284210:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761516.284219:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffff8d25d25bad80 x1876699507506688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761516.284226:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507506688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761516.284232:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761516.284234:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761516.284236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507506688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761516.284240:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761516.284251:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761516.284257:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffff8d25d25b8700 x1876699507506816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761516.284265:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507506816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761516.284270:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761516.284272:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761516.284274:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761516.284277:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761516.284286:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761516.284292:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761516/real 1789761516] req@ffff8d25d25bb100 x1876699507506944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761516.284299:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507506944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761541 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761516.284304:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761516.284305:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761516.284307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507506944:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761516.347512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761517.371557:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761518.395572:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761519.419552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761520.443586:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761521.403558:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761521.403566:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761521.403570:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761521.403579:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1499 00000100:00080000:0.0:1789761521.403589:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761521.403595:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761521.403628:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507507072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761521.403651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761521.403654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761521.403655:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761521.403658:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1499 00000100:00080000:0.0:1789761521.403661:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761521.403664:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761521.403674:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761521.403700:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507507200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761521.403708:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761521.403714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761521.403717:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761521.403717:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507507072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761521.403722:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1499 00000100:00080000:0.0:1789761521.403729:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761521.403737:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761521.403740:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761521.403755:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507507328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761521.403763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761521.403765:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761521.403767:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761521.403769:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1499 00000100:00080000:0.0:1789761521.403772:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761521.403775:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761521.403779:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761521.403791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507507456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761521.403800:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761521.403803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761521.403805:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761521.403810:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1499 00000100:00080000:0.0:1789761521.403815:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761521.403823:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761521.403830:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761521.403838:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761521.403856:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507507584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761521.403868:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761521.403871:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761521.403873:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761521.403877:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1499 00000100:00080000:0.0:1789761521.403882:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761521.403892:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761521.403893:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761521/real 1789761521] req@ffff8d25d25b9c00 x1876699507507072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761521.403906:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761521.403923:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507507712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761521.403923:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507507072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761521.404024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761521.404032:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761521.404038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507507072:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761521.404067:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761521.404092:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761521.404121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761521/real 1789761521] req@ffff8d25d25b8e00 x1876699507507200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761521.404133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507507200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761521.404140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761521.404143:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761521.404145:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761521.404150:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761521.404188:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761521.404218:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761521/real 1789761521] req@ffff8d25d25bb480 x1876699507507328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761521.404240:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507507328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761521.404250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761521.404254:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761521.404258:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761521.404282:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761521.404301:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761521.404310:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761521/real 1789761521] req@ffff8d25d25bb100 x1876699507507456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761521.404319:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507507456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761521.404325:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761521.404327:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761521.404329:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761521.404333:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507507584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761521.404344:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761521.404350:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761521/real 1789761521] req@ffff8d25d25b8700 x1876699507507584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761521.404358:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507507584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761521.404368:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761521.404370:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761521.404372:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507507584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761521.404376:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507507712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761521.404391:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761521.404402:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761521/real 1789761521] req@ffff8d25d25bad80 x1876699507507712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761521.404439:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507507712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761546 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761521.404446:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761521.404448:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761521.404450:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507507712:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761521.467538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761522.491556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761523.515536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761524.539558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761525.563518:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761526.523474:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761526.523484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761526.523487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761526.523495:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1504 00000100:00080000:2.0:1789761526.523503:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761526.523509:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761526.523548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507507840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761526.523577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761526.523581:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761526.523583:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761526.523588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1504 00000100:00080000:2.0:1789761526.523592:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761526.523597:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761526.523607:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761526.523623:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507507840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761526.523625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507507968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761526.523641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761526.523644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761526.523647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761526.523651:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1504 00000100:00080000:2.0:1789761526.523656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761526.523661:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761526.523666:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761526.523679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507508096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761526.523690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761526.523695:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761526.523697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761526.523701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761526.523707:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1504 00000100:00080000:2.0:1789761526.523712:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761526.523716:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761526.523722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761526.523739:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761526/real 1789761526] req@ffff8d25c8021880 x1876699507507840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761526.523740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507508224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761526.523775:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761526.523781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761526.523781:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507507840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761526.523785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761526.523796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1504 00000100:00080000:3.0:1789761526.523799:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761526.523801:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761526.523806:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761526.523810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761526.523814:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507507840:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761526.523819:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761526.523834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507508352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761526.523837:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761526.523855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761526.523859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761526.523861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761526.523865:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761526.523870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1504 00000100:00080000:2.0:1789761526.523877:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761526.523878:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761526/real 1789761526] req@ffff8d25c8021c00 x1876699507507968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761526.523885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761526.523895:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761526.523896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507507968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761526.523905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761526.523908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761526.523915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507507968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761526.523916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507508480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761526.523922:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761526.523945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761526.523953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761526/real 1789761526] req@ffff8d25c8021500 x1876699507508096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761526.523962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507508096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761526.523967:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761526.523969:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761526.523971:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761526.523974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761526.523985:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761526.523991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761526/real 1789761526] req@ffff8d25c8020e00 x1876699507508224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761526.523998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507508224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761526.524003:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761526.524005:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761526.524007:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761526.524016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507508352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761526.524027:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761526.524033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761526/real 1789761526] req@ffff8d25c8023480 x1876699507508352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761526.524040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507508352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761526.524045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761526.524047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761526.524049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507508352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761526.524055:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507508480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761526.524074:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761526.524085:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761526/real 1789761526] req@ffff8d25c8021880 x1876699507508480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761526.524104:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507508480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761551 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761526.524126:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761526.524130:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761526.524134:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507508480:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761526.587539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761527.611493:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761528.635562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761529.659547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761530.683569:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761531.643533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761531.643549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761531.643555:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761531.643568:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1509 00000100:00080000:2.0:1789761531.643582:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761531.643591:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761531.643645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507508608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761531.643674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761531.643677:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761531.643679:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761531.643683:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1509 00000100:00080000:2.0:1789761531.643687:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761531.643691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761531.643702:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761531.643713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507508736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761531.643726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761531.643726:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507508608:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761531.643742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761531.643746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761531.643751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1509 00000100:00080000:2.0:1789761531.643753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761531.643756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761531.643759:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761531.643769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507508864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761531.643776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761531.643780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761531.643781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761531.643784:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1509 00000100:00080000:2.0:1789761531.643793:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761531.643797:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761531.643800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761531.643808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507508992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761531.643815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761531.643819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761531.643821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761531.643826:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1509 00000100:00080000:2.0:1789761531.643830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761531.643833:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761531.643839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761531.643845:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761531.643852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507509120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761531.643861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761531.643865:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761531.643867:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761531.643870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1509 00000100:00080000:2.0:1789761531.643876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761531.643882:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508864:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761531.643883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761531.643895:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761531.643912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507509248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761531.643924:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761531.643950:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507509120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761531.643965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507509248:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761531.707504:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761532.731523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761533.755505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761534.715620:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761534.715661:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761534.715676:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761534.715712:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761534.715724:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761534.715735:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761534.715745:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761534.715756:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761534.715764:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761531/real 1789761534] req@ffff8d25c8022300 x1876699507508736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761534.715791:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507508736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761534.715809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761534.715814:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761534.715819:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761534.715827:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761531/real 1789761534] req@ffff8d25c8022d80 x1876699507508864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761534.715845:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507508864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761534.715853:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761534.715857:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761534.715861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761534.715868:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761531/real 1789761534] req@ffff8d25c8020700 x1876699507508992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761534.715881:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507508992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761534.715890:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761534.715895:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761534.715898:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507508992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761534.715905:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761531/real 1789761534] req@ffff8d25c8020000 x1876699507508608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761534.715919:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507508608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761534.715927:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761534.715931:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761534.715940:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507508608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761534.715971:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761531/real 1789761534] req@ffff8d25c8023b80 x1876699507509120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761534.715983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507509120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761534.715991:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761534.715994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761534.715997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507509120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761534.716004:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761531/real 1789761534] req@ffff8d25c8023100 x1876699507509248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761534.716016:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507509248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761556 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761534.716024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761534.716027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761534.716030:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507509248:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761534.779534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761535.803518:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761536.763512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761536.763524:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761536.763529:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761536.763542:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1514 00000100:00080000:3.0:1789761536.763554:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761536.763564:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761536.763619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507509376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761536.763662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761536.763665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761536.763667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761536.763673:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1514 00000100:00080000:3.0:1789761536.763678:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761536.763683:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761536.763695:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761536.763711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507509504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761536.763722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761536.763725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761536.763727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761536.763732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1514 00000100:00080000:3.0:1789761536.763736:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761536.763740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761536.763746:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761536.763757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507509632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761536.763773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761536.763776:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761536.763778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761536.763782:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1514 00000100:00080000:3.0:1789761536.763786:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761536.763790:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761536.763801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761536.763813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507509760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761536.763823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761536.763827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761536.763829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761536.763832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1514 00000100:00080000:3.0:1789761536.763836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761536.763840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761536.763845:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761536.763855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507509888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761536.763864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761536.763868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761536.763869:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761536.763873:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1514 00000100:00080000:3.0:1789761536.763877:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761536.763881:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761536.763887:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761536.763898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507510016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761536.764267:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507509376:192.168.203.141@tcp:250:kworker.0 00000800:00000100:0.0:1789761536.764363:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761536.764462:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761536/real 1789761536] req@ffff8d25d1b09c00 x1876699507509376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761536.764485:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507509376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761536.764499:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761536.764505:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761536.764510:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507509376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761536.764519:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507509504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761536.764546:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761536.764562:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761536/real 1789761536] req@ffff8d25d1b0ad80 x1876699507509504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761536.764576:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507509504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761536.764584:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761536.764588:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761536.764591:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507509504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761536.764597:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507509632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761536.764627:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761536.764640:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761536/real 1789761536] req@ffff8d25d1b09180 x1876699507509632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761536.764651:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507509632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761536.764659:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761536.764663:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761536.764666:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507509632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761536.764672:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507509760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761536.764702:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761536.764716:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761536/real 1789761536] req@ffff8d25d1b0aa00 x1876699507509760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761536.764728:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507509760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761536.764737:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761536.764740:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761536.764743:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507509760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761536.764755:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507509888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761536.764777:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761536.764789:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761536/real 1789761536] req@ffff8d25d1b08a80 x1876699507509888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761536.764802:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507509888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761536.764810:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761536.764814:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761536.764818:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507509888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761536.764824:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507510016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761536.764846:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761536.764859:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761536/real 1789761536] req@ffff8d25d1b0bb80 x1876699507510016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761536.764871:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507510016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761561 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761536.764879:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761536.764882:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761536.764886:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507510016:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761536.827542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761537.851522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761538.875499:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761539.899569:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761540.923555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761541.883481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761541.883496:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761541.883502:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761541.883515:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1519 00000100:00080000:3.0:1789761541.883527:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761541.883537:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761541.883587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507510144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761541.883624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761541.883629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761541.883631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761541.883636:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1519 00000100:00080000:3.0:1789761541.883641:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761541.883647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761541.883659:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761541.883668:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507510144:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761541.883676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507510272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761541.883690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761541.883695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761541.883697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761541.883702:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1519 00000100:00080000:3.0:1789761541.883706:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761541.883711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761541.883716:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761541.883730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507510400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761541.883741:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761541.883742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761541.883748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761541.883751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761541.883757:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1519 00000100:00080000:3.0:1789761541.883762:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761541.883767:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761541.883774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761541.883794:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761541/real 1789761541] req@ffff8d25d1b08700 x1876699507510144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761541.883805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507510528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761541.883820:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507510144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761541.883820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761541.883828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761541.883831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761541.883837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1519 00000100:00080000:3.0:1789761541.883844:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761541.883851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761541.883854:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761541.883859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761541.883860:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761541.883866:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507510144:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761541.883877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507510656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761541.883889:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761541.883894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761541.883897:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507510272:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761541.883898:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761541.883908:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1519 00000100:00080000:3.0:1789761541.883912:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761541.883917:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761541.883923:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761541.883924:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761541.883939:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761541/real 1789761541] req@ffff8d25d1b0b800 x1876699507510272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761541.883944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507510784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761541.883960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507510272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761541.883969:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761541.883973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761541.883976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507510272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761541.883980:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507510400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761541.883994:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761541.884021:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761541/real 1789761541] req@ffff8d25d1b08e00 x1876699507510400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761541.884043:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507510400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761541.884052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761541.884056:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761541.884060:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507510400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761541.884068:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507510528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761541.884087:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761541.884101:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761541/real 1789761541] req@ffff8d25d1b0a680 x1876699507510528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761541.884113:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507510528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761541.884122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761541.884126:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761541.884130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507510528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761541.884137:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507510656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761541.884159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761541.884173:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761541/real 1789761541] req@ffff8d25d1b09880 x1876699507510656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761541.884187:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507510656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761541.884196:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761541.884200:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761541.884203:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507510656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761541.884229:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507510784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761541.884252:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761541.884266:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761541/real 1789761541] req@ffff8d25d1b08700 x1876699507510784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761541.884278:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507510784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761566 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761541.884287:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761541.884291:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761541.884295:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507510784:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761541.947537:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761542.971516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761543.995609:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761545.019544:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761546.043506:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761547.003496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761547.003505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761547.003510:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761547.003520:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1524 00000100:00080000:3.0:1789761547.003530:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761547.003538:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761547.003575:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507510912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761547.003603:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761547.003607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761547.003609:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761547.003615:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1524 00000100:00080000:3.0:1789761547.003620:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761547.003625:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761547.003634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507510912:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789761547.003636:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761547.003657:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507511040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761547.003670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761547.003674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761547.003676:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761547.003680:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1524 00000100:00080000:3.0:1789761547.003685:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761547.003689:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761547.003693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761547.003697:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761547.003712:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507511168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761547.003731:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761547.003735:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffff8d25d1b0bb80 x1876699507510912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761547.003736:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761547.003740:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761547.003748:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1524 00000100:00080000:3.0:1789761547.003756:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761547.003762:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761547.003767:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761547.003769:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507510912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761547.003779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507511296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761547.003785:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761547.003788:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761547.003791:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761547.003797:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789761547.003798:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507510912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761547.003802:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761547.003809:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1524 00000100:00080000:3.0:1789761547.003814:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761547.003818:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761547.003823:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511040:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789761547.003824:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761547.003843:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507511424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761547.003855:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761547.003855:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761547.003861:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761547.003864:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761547.003869:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1524 00000100:00100000:0.0:1789761547.003875:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffff8d25d1b08a80 x1876699507511040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761547.003876:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761547.003885:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761547.003893:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761547.003895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507511040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.003904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.003907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761547.003911:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507511552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761547.003912:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761547.003922:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761547.003943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761547.003954:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffff8d25d1b0aa00 x1876699507511168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761547.003977:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507511168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.003988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.003990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761547.003992:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761547.003996:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761547.004012:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761547.004023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffff8d25d1b09180 x1876699507511296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761547.004035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507511296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.004044:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.004048:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761547.004051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761547.004075:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507511424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761547.004094:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761547.004103:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffff8d25d1b0ad80 x1876699507511424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761547.004111:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507511424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.004116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.004118:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761547.004121:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507511424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761547.004124:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507511552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761547.004135:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761547.004141:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761547/real 1789761547] req@ffff8d25d1b09c00 x1876699507511552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761547.004148:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507511552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761572 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761547.004153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761547.004155:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761547.004157:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507511552:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761547.067523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761548.091546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761549.115487:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761550.139537:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761551.163514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761552.123464:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761552.123477:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761552.123483:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761552.123497:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1529 00000100:00080000:3.0:1789761552.123509:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761552.123520:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761552.123566:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507511680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761552.123600:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761552.123605:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761552.123607:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761552.123612:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1529 00000100:00080000:3.0:1789761552.123617:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761552.123622:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761552.123634:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761552.123649:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507511680:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761552.123651:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507511808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761552.123667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761552.123669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761552.123671:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761552.123675:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1529 00000100:00080000:3.0:1789761552.123679:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761552.123682:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761552.123687:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761552.123698:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507511936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761552.123709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761552.123714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761552.123716:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761552.123721:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1529 00000100:00080000:3.0:1789761552.123726:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761552.123731:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761552.123739:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789761552.123748:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761552.123801:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761552] req@ffff8d25d1b0b100 x1876699507511680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761552.123827:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507512064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761552.123830:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507511680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761552.123858:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761552.123863:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761552.123867:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507511680:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761552.123898:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761552.123944:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761552.123962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761552] req@ffff8d25d1b0b480 x1876699507511808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761552.123987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507511808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761552.123987:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761552.123998:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761552.124003:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761552.124013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761552.124014:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1529 00000100:00080000:3.0:1789761552.124023:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761552.124024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761552.124034:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761552.124036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761552.124049:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761552.124079:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761552.124090:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761552.124098:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761552] req@ffff8d25d1b09500 x1876699507511936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761552.124108:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507512192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761552.124119:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507511936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761552.124128:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761552.124135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761552.124135:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761552.124140:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761552.124144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761552.124150:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1529 00000100:00100000:2.0:1789761552.124151:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507511936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761552.124159:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761552.124164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512064:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789761552.124169:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761552.124177:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761552.124192:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507512320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761552.124200:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761552.124222:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761552] req@ffff8d25d1b08000 x1876699507512064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761552.124244:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507512064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761552.124251:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761552.124254:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761552.124256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761552.124274:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507512192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761552.124290:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761552.124299:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761552] req@ffff8d25d1b08380 x1876699507512192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761552.124307:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507512192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761552.124313:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761552.124315:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761552.124317:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507512192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761552.124321:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507512320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761552.124331:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761552.124337:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761552/real 1789761552] req@ffff8d25d1b0a300 x1876699507512320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761552.124344:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507512320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761577 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761552.124354:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761552.124356:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761552.124359:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507512320:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761552.187562:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761553.211496:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761554.235547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761555.259487:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761556.283519:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761557.243513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761557.243532:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761557.243537:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761557.243548:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1534 00000100:00080000:0.0:1789761557.243559:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761557.243566:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761557.243611:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507512448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761557.243637:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761557.243643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761557.243644:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761557.243647:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1534 00000100:00080000:0.0:1789761557.243650:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761557.243653:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761557.243663:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761557.243672:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507512576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761557.243685:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507512448:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761557.243688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761557.243703:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761557.243706:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761557.243710:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1534 00000100:00080000:0.0:1789761557.243713:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761557.243716:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761557.243719:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761557.243728:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507512704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761557.243735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761557.243739:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761557.243740:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761557.243744:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1534 00000100:00080000:0.0:1789761557.243746:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761557.243749:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761557.243755:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789761557.243761:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761557.243766:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507512832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761557.243776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761557.243779:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761557.243781:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761557.243786:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1534 00000100:00080000:0.0:1789761557.243788:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761557.243791:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761557.243795:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761557.243808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507512960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761557.243809:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffff8d25d25b9f80 x1876699507512448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761557.243861:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761557.243866:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761557.243868:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761557.243873:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1534 00000100:00080000:0.0:1789761557.243876:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761557.243881:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761557.243896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507512448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761557.243899:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761557.243918:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507513088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761557.243920:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761557.243936:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761557.243943:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507512448:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761557.243970:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761557.243992:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761557.244001:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffff8d25d25bb800 x1876699507512576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761557.244010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507512576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761557.244016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761557.244018:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761557.244020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761557.244025:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761557.244037:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761557.244044:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffff8d25d25b8380 x1876699507512704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761557.244052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507512704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761557.244057:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761557.244059:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761557.244061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761557.244065:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761557.244075:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761557.244081:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffff8d25d25ba680 x1876699507512832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761557.244088:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507512832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761557.244093:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761557.244095:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761557.244097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507512832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761557.244101:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507512960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761557.244111:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761557.244117:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffff8d25d25b9180 x1876699507512960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761557.244125:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507512960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761557.244130:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761557.244132:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761557.244134:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507512960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761557.244137:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507513088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761557.244148:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761557.244154:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761557/real 1789761557] req@ffff8d25d25b8a80 x1876699507513088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761557.244162:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507513088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761582 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761557.244167:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761557.244168:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761557.244170:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507513088:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761557.307533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761558.331539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761559.355592:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761560.379524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761561.403531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761562.363513:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761562.363539:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761562.363546:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761562.363562:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1540 00000100:00080000:3.0:1789761562.363574:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761562.363582:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761562.363635:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507513216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761562.363663:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761562.363667:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761562.363669:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761562.363677:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1540 00000100:00080000:3.0:1789761562.363682:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761562.363687:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761562.363698:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761562.363713:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507513344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761562.363735:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507513216:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761562.363743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761562.363748:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761562.363750:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761562.363755:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1540 00000100:00080000:3.0:1789761562.363766:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761562.363770:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761562.363776:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761562.363791:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507513472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761562.363802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789761562.363805:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761562.363824:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761562.363826:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761562.363831:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1540 00000100:00080000:3.0:1789761562.363836:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761562.363840:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761562.363858:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761562.363866:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffff8d25d1b0a300 x1876699507513216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761562.363873:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507513600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761562.363915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761562.363917:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761562.363923:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761562.363924:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507513216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761562.363939:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1540 00000100:00080000:3.0:1789761562.363952:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761562.363957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761562.363965:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789761562.363966:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761562.363973:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507513216:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789761562.363978:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761562.363995:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507513728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761562.364000:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507513344:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761562.364026:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761562.364030:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761562.364032:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761562.364037:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1540 00000100:00080000:3.0:1789761562.364042:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761562.364049:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761562.364053:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761562.364058:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761562.364073:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507513856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761562.364082:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffff8d25d1b08380 x1876699507513344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.364098:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507513344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.364106:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.364109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761562.364112:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507513344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761562.364116:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507513472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761562.364130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761562.364137:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffff8d25d1b08000 x1876699507513472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.364144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507513472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.364150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.364152:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761562.364154:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507513472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761562.364159:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507513600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761562.364169:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761562.364175:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffff8d25d1b09500 x1876699507513600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.364182:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507513600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.364188:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.364189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761562.364191:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507513600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761562.364205:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507513728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761562.364215:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761562.364222:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffff8d25d1b0b480 x1876699507513728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.364229:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507513728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.364234:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.364236:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761562.364238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507513728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761562.364241:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507513856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761562.364252:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761562.364258:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761562/real 1789761562] req@ffff8d25d1b0b100 x1876699507513856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761562.364265:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507513856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761587 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761562.364270:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761562.364272:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761562.364274:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507513856:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761562.427504:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761563.451508:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761564.475490:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761565.499569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761566.523521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761567.483541:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761567.483554:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761567.483559:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761567.483571:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1545 00000100:00080000:0.0:1789761567.483583:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761567.483592:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761567.483643:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507513984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761567.483672:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761567.483675:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761567.483677:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761567.483680:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1545 00000100:00080000:0.0:1789761567.483684:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761567.483687:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761567.483698:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761567.483708:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507514112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761567.483721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761567.483725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789761567.483725:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507513984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761567.483730:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761567.483737:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1545 00000100:00080000:0.0:1789761567.483741:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761567.483745:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761567.483752:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761567.483768:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507514240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761567.483780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761567.483783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761567.483784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761567.483788:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1545 00000100:00080000:0.0:1789761567.483792:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761567.483795:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761567.483799:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789761567.483804:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761567.483828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507514368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761567.483842:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761567.483846:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761567.483848:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761567.483853:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1545 00000100:00080000:0.0:1789761567.483858:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761567.483863:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761567.483869:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761567.483870:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761567/real 1789761567] req@ffff8d25d25ba300 x1876699507513984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761567.483882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507514496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761567.483891:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507513984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761567.483893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761567.483901:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761567.483904:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761567.483911:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1545 00000100:00080000:2.0:1789761567.483913:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761567.483915:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761567.483922:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789761567.483922:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789761567.483926:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761567.483931:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507513984:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761567.483937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507514624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761567.483958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761567.483980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761567.483989:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761567/real 1789761567] req@ffff8d25d25b8000 x1876699507514112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761567.483998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507514112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761567.484009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761567.484012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761567.484014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761567.484018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761567.484029:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761567.484035:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761567/real 1789761567] req@ffff8d25d25bbb80 x1876699507514240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761567.484042:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507514240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761567.484048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761567.484050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761567.484052:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761567.484055:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761567.484067:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761567.484073:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761567/real 1789761567] req@ffff8d25d25b8a80 x1876699507514368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761567.484080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507514368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761567.484086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761567.484087:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761567.484090:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761567.484094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507514496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761567.484105:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761567.484111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761567/real 1789761567] req@ffff8d25d25b9180 x1876699507514496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761567.484118:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507514496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761567.484123:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761567.484125:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761567.484127:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507514496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761567.484130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507514624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761567.484140:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761567.484146:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761567/real 1789761567] req@ffff8d25d25ba680 x1876699507514624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761567.484154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507514624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761592 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761567.484159:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761567.484161:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761567.484163:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507514624:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761567.547580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761568.571538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761569.595507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761570.619500:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761571.643568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761572.603468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761572.603476:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761572.603480:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761572.603488:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1550 00000100:00080000:3.0:1789761572.603496:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761572.603502:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761572.603529:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507514752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761572.603550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761572.603552:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761572.603554:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761572.603557:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1550 00000100:00080000:3.0:1789761572.603560:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761572.603563:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761572.603571:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761572.603580:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507514752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761572.603581:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507514880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761572.603596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761572.603599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761572.603601:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761572.603605:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1550 00000100:00080000:3.0:1789761572.603610:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761572.603615:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761572.603621:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761572.603634:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507515008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761572.603641:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761572.603654:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761572.603657:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761572.603658:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761572.603662:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1550 00000100:00080000:3.0:1789761572.603666:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761572.603669:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761572.603674:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761572.603679:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761572/real 1789761572] req@ffff8d25d1b09c00 x1876699507514752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761572.603690:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507515136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761572.603696:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507514752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761572.603711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761572.603720:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761572.603724:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761572.603726:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761572.603731:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1550 00000100:00080000:2.0:1789761572.603732:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761572.603738:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761572.603739:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507514752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761572.603746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761572.603752:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761572.603761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507515264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761572.603767:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514880:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761572.603795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761572.603798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761572.603800:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761572.603804:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1550 00000100:00080000:3.0:1789761572.603809:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761572.603813:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789761572.603819:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761572.603820:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761572.603832:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507515392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761572.603841:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761572/real 1789761572] req@ffff8d25d1b0ad80 x1876699507514880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761572.603858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507514880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761572.603868:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761572.603870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761572.603873:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507514880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761572.603877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761572.603895:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761572.603903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761572/real 1789761572] req@ffff8d25d1b09180 x1876699507515008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761572.603911:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507515008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761572.603921:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761572.603923:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761572.603925:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761572.603929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761572.603939:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761572.603945:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761572/real 1789761572] req@ffff8d25d1b0aa00 x1876699507515136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761572.603952:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507515136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761572.603958:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761572.603959:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761572.603961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761572.603974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507515264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761572.603985:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761572.603991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761572/real 1789761572] req@ffff8d25d1b08a80 x1876699507515264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761572.603999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507515264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761572.604004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761572.604006:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761572.604008:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507515264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761572.604012:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507515392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761572.604022:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761572.604028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761572/real 1789761572] req@ffff8d25d1b0bb80 x1876699507515392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761572.604035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507515392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761597 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761572.604040:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761572.604041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761572.604043:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507515392:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761572.667503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761573.691534:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761574.715484:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761575.739482:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761576.763545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761577.723433:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761577.723441:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761577.723445:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761577.723455:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1555 00000100:00080000:3.0:1789761577.723465:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761577.723472:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761577.723511:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507515520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761577.723532:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761577.723534:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761577.723537:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761577.723540:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1555 00000100:00080000:3.0:1789761577.723543:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761577.723547:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761577.723555:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761577.723566:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507515648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761577.723573:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507515520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761577.723577:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761577.723583:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761577.723584:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761577.723587:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1555 00000100:00080000:3.0:1789761577.723589:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761577.723591:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761577.723601:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761577.723611:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507515776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761577.723618:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761577.723620:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761577.723622:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761577.723625:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1555 00000100:00080000:3.0:1789761577.723628:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761577.723630:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761577.723634:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789761577.723635:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761577.723647:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507515904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761577.723660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761577.723663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761577.723665:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761577.723669:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1555 00000100:00080000:3.0:1789761577.723673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761577.723679:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761577/real 1789761577] req@ffff8d25d1b08700 x1876699507515520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761577.723681:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761577.723692:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761577.723699:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507515520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761577.723709:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761577.723710:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507516032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761577.723714:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761577.723722:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507515520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761577.723742:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761577.723745:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761577.723747:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761577.723752:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1555 00000100:00080000:3.0:1789761577.723756:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761577.723761:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761577.723765:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761577.723766:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761577.723780:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507516160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789761577.723794:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761577.723807:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761577/real 1789761577] req@ffff8d25d1b09880 x1876699507515648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761577.723816:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507515648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761577.723822:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761577.723824:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761577.723827:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761577.723831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761577.723842:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761577.723849:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761577/real 1789761577] req@ffff8d25d1b0a680 x1876699507515776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761577.723857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507515776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761577.723862:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761577.723864:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761577.723865:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761577.723869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761577.723879:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761577.723885:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761577/real 1789761577] req@ffff8d25d1b08e00 x1876699507515904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761577.723893:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507515904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761577.723898:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761577.723900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761577.723902:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507515904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761577.723906:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761577.723915:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761577.723921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761577/real 1789761577] req@ffff8d25d1b0b800 x1876699507516032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761577.723928:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507516032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761577.723933:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761577.723935:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761577.723937:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761577.723951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761577.723961:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761577.723967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761577/real 1789761577] req@ffff8d25d1b08700 x1876699507516160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761577.723974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507516160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761602 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761577.723979:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761577.723981:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761577.723984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516160:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761577.787524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761578.811511:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761579.835520:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761580.859502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761581.883480:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761582.843479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761582.843487:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761582.843491:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761582.843503:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1560 00000100:00080000:2.0:1789761582.843514:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761582.843522:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761582.843560:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507516288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761582.843590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761582.843593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761582.843595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761582.843598:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1560 00000100:00080000:2.0:1789761582.843602:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761582.843605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761582.843614:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761582.843635:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507516416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761582.843648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761582.843650:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507516288:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761582.843653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761582.843658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761582.843664:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1560 00000100:00080000:2.0:1789761582.843669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761582.843674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761582.843679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761582.843691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507516544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761582.843712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761582.843714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761582.843715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761582.843718:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1560 00000100:00080000:2.0:1789761582.843722:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761582.843724:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761582.843729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789761582.843734:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761582.843741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507516672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761582.843750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761582.843752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761582.843753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761582.843756:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1560 00000100:00080000:2.0:1789761582.843758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761582.843760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761582.843764:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761582.843773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507516800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761582.843777:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761582/real 1789761582] req@ffff8d25c8020380 x1876699507516288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761582.843800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761582.843805:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507516288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761582.843808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761582.843813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761582.843822:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761582.843823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1560 00000100:00080000:0.0:1789761582.843828:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761582.843831:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761582.843832:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507516288:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761582.843841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761582.843848:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761582.843858:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507516416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761582.843864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507516928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761582.843882:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761582.843894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761582/real 1789761582] req@ffff8d25c8022a00 x1876699507516416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761582.843904:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507516416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761582.843910:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761582.843912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761582.843914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507516416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761582.843918:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507516544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761582.843929:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761582.843935:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761582/real 1789761582] req@ffff8d25c8022680 x1876699507516544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761582.843943:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507516544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761582.843948:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761582.843950:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761582.843951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507516544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761582.843955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507516672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761582.843966:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761582.843972:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761582/real 1789761582] req@ffff8d25c8021180 x1876699507516672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761582.843980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507516672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761582.843990:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761582.843992:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761582.843994:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507516672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761582.843997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761582.844007:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761582.844013:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761582/real 1789761582] req@ffff8d25c8020a80 x1876699507516800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761582.844020:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507516800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761582.844026:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761582.844027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761582.844029:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761582.844040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761582.844050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761582.844056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761582/real 1789761582] req@ffff8d25c8023800 x1876699507516928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761582.844064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507516928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761607 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761582.844069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761582.844071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761582.844073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507516928:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761582.907559:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761583.931498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761584.955536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761585.979498:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761587.003551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761587.963472:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761587.963483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761587.963487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761587.963495:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1565 00000100:00080000:2.0:1789761587.963503:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761587.963509:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761587.963545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507517056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761587.963578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761587.963583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761587.963586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761587.963593:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1565 00000100:00080000:2.0:1789761587.963598:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761587.963604:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761587.963610:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507517056:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761587.963616:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761587.963637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507517184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761587.963652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761587.963672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761587.963675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761587.963680:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1565 00000800:00000100:3.0:1789761587.963685:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761587.963689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761587.963698:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761587.963704:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761587.963723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507517312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761587.963733:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761587/real 1789761587] req@ffff8d25c8023800 x1876699507517056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761587.963737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761587.963756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761587.963758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761587.963764:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1565 00000100:00080000:2.0:1789761587.963768:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761587.963773:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761587.963779:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761587.963791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507517440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761587.963799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761587.963802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761587.963803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761587.963806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1565 00000100:00080000:2.0:1789761587.963809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761587.963812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761587.963815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761587.963822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507517568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761587.963828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761587.963831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761587.963833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761587.963835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1565 00000100:00080000:2.0:1789761587.963837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761587.963840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761587.963842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761587.963845:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507517056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761587.963851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507517696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761587.963957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761587.963960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761587.963964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507517056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761587.963986:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761587.964055:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761587.964066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761587/real 1789761587] req@ffff8d25c8020a80 x1876699507517184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761587.964075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507517184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761587.964081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761587.964084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761587.964086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761587.964090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761587.964101:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761587.964108:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761587/real 1789761587] req@ffff8d25c8021180 x1876699507517312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761587.964116:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507517312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761587.964121:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761587.964123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761587.964125:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761587.964129:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761587.964139:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761587.964145:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761587/real 1789761587] req@ffff8d25c8022680 x1876699507517440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761587.964152:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507517440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761587.964157:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761587.964159:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761587.964160:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761587.964164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507517568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761587.964173:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761587.964179:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761587/real 1789761587] req@ffff8d25c8022a00 x1876699507517568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761587.964185:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507517568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761587.964191:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761587.964193:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761587.964195:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507517568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761587.964198:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507517696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761587.964207:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761587.964213:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761587/real 1789761587] req@ffff8d25c8020380 x1876699507517696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761587.964220:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507517696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761612 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761587.964225:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761587.964227:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761587.964229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507517696:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761588.027486:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761589.051518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761590.075469:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761591.099543:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761592.123509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761593.083465:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761593.083475:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761593.083479:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761593.083489:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1570 00000100:00080000:3.0:1789761593.083498:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761593.083505:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761593.083546:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507517824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761593.083569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761593.083572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761593.083573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761593.083577:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1570 00000100:00080000:3.0:1789761593.083581:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761593.083585:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761593.083593:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761593.083602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507517824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761593.083603:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507517952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761593.083630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761593.083633:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761593.083634:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761593.083637:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1570 00000100:00080000:3.0:1789761593.083640:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761593.083643:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761593.083647:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761593.083657:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507518080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761593.083663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761593.083666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761593.083667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761593.083670:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1570 00000100:00080000:3.0:1789761593.083672:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761593.083677:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761593.083678:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761593.083686:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761593.083697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507518208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761593.083704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761593.083706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761593.083708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761593.083710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1570 00000100:00080000:3.0:1789761593.083713:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761593.083718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761593.083719:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffff8d25d1b0bb80 x1876699507517824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761593.083722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761593.083734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507518336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761593.083742:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507517824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761593.083743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761593.083751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761593.083754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761593.083761:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1570 00000100:00080000:0.0:1789761593.083763:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761593.083764:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761593.083770:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761593.083773:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789761593.083775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761593.083779:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507517824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761593.083787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507518464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761593.083803:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761593.083821:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761593.083831:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffff8d25d1b08a80 x1876699507517952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761593.083840:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507517952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761593.083846:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761593.083849:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761593.083851:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507517952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761593.083857:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761593.083868:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761593.083875:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffff8d25d1b0aa00 x1876699507518080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761593.083883:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507518080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761593.083888:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761593.083890:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761593.083892:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761593.083896:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761593.083907:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761593.083913:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffff8d25d1b09180 x1876699507518208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761593.083920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507518208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761593.083926:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761593.083927:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761593.083929:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761593.083932:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507518336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761593.083942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761593.083949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffff8d25d1b0ad80 x1876699507518336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761593.083957:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507518336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761593.083963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761593.083964:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761593.083966:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507518336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761593.083969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507518464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761593.083980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761593.083986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761593/real 1789761593] req@ffff8d25d1b09c00 x1876699507518464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761593.083996:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507518464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761618 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761593.084005:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761593.084008:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761593.084012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507518464:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761593.147501:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761594.171553:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761595.195516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761596.219523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761597.243481:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761598.203480:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761598.203489:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761598.203494:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761598.203504:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1575 00000100:00080000:0.0:1789761598.203514:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761598.203521:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761598.203560:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507518592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761598.203597:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761598.203600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761598.203602:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761598.203605:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1575 00000100:00080000:0.0:1789761598.203609:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761598.203612:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761598.203620:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761598.203632:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507518720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761598.203640:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761598.203644:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761598.203644:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507518592:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761598.203647:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761598.203654:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1575 00000100:00080000:0.0:1789761598.203664:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761598.203667:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761598.203673:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761598.203681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507518848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761598.203688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761598.203691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761598.203693:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761598.203696:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1575 00000100:00080000:0.0:1789761598.203699:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761598.203702:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761598.203705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761598.203716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507518976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761598.203727:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761598.203730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761598.203733:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761598.203737:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1575 00000100:00080000:0.0:1789761598.203741:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761598.203746:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761598.203753:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761598.203769:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507519104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761598.203769:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518720:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761598.203787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761598.203790:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761598.203792:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761598.203797:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1575 00000100:00080000:0.0:1789761598.203803:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761598.203808:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761598.203813:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518848:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789761598.203815:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761598.203833:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507519232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761598.203839:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761598.203866:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507519104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761598.203881:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507519232:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761598.267522:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761599.291512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761600.315493:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789761601.275631:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789761601.275667:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789761601.275682:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761601.275720:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761601.275734:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761601.275763:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761601.275772:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761601] req@ffff8d25d25bb800 x1876699507518720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1789761601.275776:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789761601.275793:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761601.275818:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507518720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761601.275831:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761601.275836:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761601.275841:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761601.275849:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761601] req@ffff8d25d25b9f80 x1876699507518848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761601.275869:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507518848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761601.275878:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761601.275883:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761601.275887:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761601.275894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761601] req@ffff8d25d25bad80 x1876699507518976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761601.275907:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507518976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761601.275915:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761601.275919:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761601.275922:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507518976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761601.275930:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761601] req@ffff8d25d25b8380 x1876699507518592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761601.275944:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507518592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761601.275953:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761601.275957:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761601.275961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507518592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761601.275989:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761601] req@ffff8d25d25b8700 x1876699507519104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761601.276003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507519104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761601.276012:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761601.276015:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761601.276019:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507519104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761601.276027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761598/real 1789761601] req@ffff8d25d25bb100 x1876699507519232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761601.276041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507519232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761623 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761601.276050:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761601.276053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761601.276057:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507519232:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761601.339514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761602.363585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761603.323510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.323522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761603.323526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.323550:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1581 00000100:00080000:2.0:1789761603.323563:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.323571:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761603.323612:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507519360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761603.323635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.323639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761603.323641:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.323644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1581 00000100:00080000:2.0:1789761603.323647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.323650:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761603.323660:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761603.323670:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507519488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761603.323680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.323683:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761603.323684:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507519360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761603.323685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.323692:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1581 00000100:00080000:2.0:1789761603.323697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.323700:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761603.323703:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761603.323711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507519616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761603.323717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.323720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761603.323721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.323724:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1581 00000100:00080000:2.0:1789761603.323727:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.323731:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761603.323737:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761603.323749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507519744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761603.323759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.323763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761603.323764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.323768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1581 00000100:00080000:2.0:1789761603.323772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.323778:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761603.323780:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761603.323786:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761603.323800:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507519872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761603.323810:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761603.323814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761603.323816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761603.323821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1581 00000100:00080000:2.0:1789761603.323825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761603.323829:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761603.323835:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761603.323839:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffff8d25c8023100 x1876699507519360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761603.323851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507520000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761603.323870:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507519360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761603.323885:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761603.323891:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761603.323897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507519360:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761603.323930:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507519488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761603.323959:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761603.323975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffff8d25c8023b80 x1876699507519488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761603.323989:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507519488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761603.323999:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761603.324003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761603.324007:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507519488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761603.324014:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507519616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761603.324035:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761603.324048:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffff8d25c8020000 x1876699507519616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761603.324061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507519616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761603.324069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761603.324073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761603.324077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507519616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761603.324084:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507519744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761603.324106:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761603.324119:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffff8d25c8020700 x1876699507519744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761603.324132:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507519744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761603.324141:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761603.324144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761603.324148:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507519744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761603.324155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507519872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761603.324178:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761603.324198:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffff8d25c8022d80 x1876699507519872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761603.324212:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507519872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761603.324220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761603.324224:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761603.324227:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507519872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761603.324235:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507520000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761603.324344:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761603.324359:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761603/real 1789761603] req@ffff8d25c8022300 x1876699507520000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761603.324373:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507520000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761628 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761603.324409:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761603.324413:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761603.324417:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507520000:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761603.387516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761604.411571:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761605.435573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761606.459615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761607.483497:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761608.443495:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761608.443519:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761608.443525:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761608.443537:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1586 00000100:00080000:2.0:1789761608.443548:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761608.443556:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761608.443597:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507520128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761608.443623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761608.443626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761608.443627:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761608.443632:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1586 00000100:00080000:2.0:1789761608.443635:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761608.443639:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761608.443649:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761608.443664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507520256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761608.443666:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507520128:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761608.443692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761608.443695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761608.443697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761608.443701:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1586 00000100:00080000:2.0:1789761608.443703:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761608.443706:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761608.443709:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761608.443718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507520384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761608.443738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761608.443747:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761608.443749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761608.443752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761608.443757:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1586 00000100:00080000:2.0:1789761608.443761:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761608.443764:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761608.443770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761608.443785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507520512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761608.443793:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761608/real 1789761608] req@ffff8d25c8021880 x1876699507520128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761608.443796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761608.443806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761608.443808:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761608.443811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1586 00000100:00080000:2.0:1789761608.443820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761608.443832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761608.443839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761608.443840:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507520128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761608.443858:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761608.443866:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761608.443868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507520640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761608.443871:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507520128:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761608.443892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761608.443897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761608.443900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761608.443908:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1586 00000100:00100000:3.0:1789761608.443909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507520256:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761608.443920:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761608.443929:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761608.443936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761608.443937:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761608.443951:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761608/real 1789761608] req@ffff8d25c8023480 x1876699507520256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761608.443960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507520768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761608.443980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507520256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761608.443993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761608.443996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761608.443999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507520256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761608.444005:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507520384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761608.444036:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761608.444050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761608/real 1789761608] req@ffff8d25c8020e00 x1876699507520384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761608.444063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507520384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761608.444072:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761608.444075:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761608.444079:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507520384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761608.444086:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507520512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761608.444109:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761608.444123:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761608/real 1789761608] req@ffff8d25c8021500 x1876699507520512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761608.444137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507520512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761608.444146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761608.444150:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761608.444154:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507520512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761608.444161:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507520640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761608.444184:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761608.444198:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761608/real 1789761608] req@ffff8d25c8021c00 x1876699507520640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761608.444211:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507520640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761608.444220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761608.444223:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761608.444227:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507520640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761608.444245:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507520768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761608.444261:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761608.444270:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761608/real 1789761608] req@ffff8d25c8021880 x1876699507520768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761608.444277:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507520768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761633 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761608.444283:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761608.444285:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761608.444287:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507520768:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761608.507520:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761609.531512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761610.555534:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761611.579513:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761612.603538:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761613.563536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761613.563551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761613.563556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761613.563570:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1591 00000100:00080000:2.0:1789761613.563581:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761613.563590:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761613.563633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507520896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761613.563661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761613.563664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761613.563665:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761613.563669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1591 00000100:00080000:2.0:1789761613.563672:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761613.563675:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761613.563686:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761613.563695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507521024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761613.563705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761613.563709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761613.563712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761613.563719:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507520896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761613.563721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1591 00000100:00080000:2.0:1789761613.563726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761613.563737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761613.563742:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761613.563752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507521152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761613.563759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761613.563762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761613.563763:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761613.563766:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1591 00000100:00080000:2.0:1789761613.563768:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761613.563771:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761613.563774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761613.563782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507521280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761613.563790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761613.563793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761613.563795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761613.563797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1591 00000100:00080000:2.0:1789761613.563802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761613.563809:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761613.563818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761613.563824:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761613.563836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507521408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761613.563844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761613.563847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761613.563850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761613.563858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1591 00000100:00100000:3.0:1789761613.563863:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761613/real 1789761613] req@ffff8d25c8022300 x1876699507520896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761613.563866:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761613.563876:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761613.563883:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761613.563902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507521536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761613.563906:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507520896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761613.563954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761613.563962:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761613.563970:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507520896:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761613.563997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761613.564018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761613.564028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761613/real 1789761613] req@ffff8d25c8022d80 x1876699507521024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761613.564038:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507521024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761613.564044:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761613.564046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761613.564048:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761613.564052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761613.564064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761613.564071:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761613/real 1789761613] req@ffff8d25c8020700 x1876699507521152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761613.564079:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507521152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761613.564095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761613.564102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761613.564106:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761613.564112:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761613.564135:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761613.564148:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761613/real 1789761613] req@ffff8d25c8020000 x1876699507521280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761613.564158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507521280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761613.564163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761613.564166:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761613.564168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761613.564172:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507521408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761613.564187:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761613.564195:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761613/real 1789761613] req@ffff8d25c8023b80 x1876699507521408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761613.564202:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507521408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761613.564212:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761613.564215:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761613.564217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507521408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761613.564220:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507521536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761613.564231:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761613.564237:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761613/real 1789761613] req@ffff8d25c8023100 x1876699507521536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761613.564244:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507521536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761638 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761613.564249:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761613.564251:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761613.564252:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507521536:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761613.627514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761614.651545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761615.675542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761616.699514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761617.723523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761618.683471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761618.683483:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761618.683487:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761618.683497:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1596 00000100:00080000:0.0:1789761618.683506:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761618.683514:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761618.683548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507521664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761618.683569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761618.683572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761618.683573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761618.683577:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1596 00000100:00080000:0.0:1789761618.683580:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761618.683583:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761618.683591:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761618.683600:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507521792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761618.683609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761618.683611:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507521664:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761618.683612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761618.683616:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761618.683624:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1596 00000100:00080000:0.0:1789761618.683626:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761618.683628:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761618.683631:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761618.683644:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507521920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761618.683654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761618.683658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761618.683660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761618.683664:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1596 00000100:00080000:0.0:1789761618.683668:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761618.683672:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761618.683678:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761618.683692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507522048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761618.683697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761618.683707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761618.683710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761618.683712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761618.683716:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1596 00000100:00080000:0.0:1789761618.683720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761618.683723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761618.683727:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761618.683737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507522176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761618.683746:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761618/real 1789761618] req@ffff8d25d25bb480 x1876699507521664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761618.683747:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761618.683755:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761618.683758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761618.683766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1596 00000100:00080000:0.0:1789761618.683768:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761618.683771:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761618.683774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761618.683777:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507521664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761618.683784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507522304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761618.683813:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761618.683818:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761618.683823:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507521664:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761618.683846:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761618.683869:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761618.683881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761618/real 1789761618] req@ffff8d25d25b8e00 x1876699507521792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761618.683894:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507521792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761618.683903:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761618.683907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761618.683910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761618.683917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761618.683933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761618.683943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761618/real 1789761618] req@ffff8d25d25b9c00 x1876699507521920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761618.683955:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507521920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761618.683963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761618.683966:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761618.683969:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507521920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761618.683976:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761618.683992:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761618.684002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761618/real 1789761618] req@ffff8d25d25bb100 x1876699507522048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761618.684013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507522048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761618.684030:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761618.684034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761618.684038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761618.684044:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507522176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761618.684061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761618.684071:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761618/real 1789761618] req@ffff8d25d25b8700 x1876699507522176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761618.684083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507522176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761618.684091:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761618.684094:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761618.684097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507522176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761618.684104:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507522304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761618.684119:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761618.684129:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761618/real 1789761618] req@ffff8d25d25b8380 x1876699507522304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761618.684141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507522304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761643 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761618.684149:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761618.684152:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761618.684155:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507522304:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761618.747523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761619.771607:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761620.795555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761621.819588:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761622.843539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761623.803486:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761623.803497:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761623.803501:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761623.803519:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1601 00000100:00080000:0.0:1789761623.803528:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761623.803535:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761623.803567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507522432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761623.803588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761623.803592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761623.803594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761623.803599:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1601 00000100:00080000:0.0:1789761623.803603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761623.803607:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761623.803617:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761623.803632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507522560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761623.803635:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507522432:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761623.803642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761623.803646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761623.803647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761623.803651:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1601 00000100:00080000:0.0:1789761623.803654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761623.803658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761623.803662:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761623.803673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507522688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761623.803681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761623.803685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761623.803687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761623.803690:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1601 00000100:00080000:0.0:1789761623.803694:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761623.803697:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761623.803701:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761623.803703:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761623.803711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507522816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761623.803720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761623.803722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761623.803724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761623.803727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1601 00000100:00080000:0.0:1789761623.803730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761623.803734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761623.803738:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761623.803740:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761623/real 1789761623] req@ffff8d25d25bad80 x1876699507522432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761623.803748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507522944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761623.803756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761623.803758:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507522432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761623.803759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761623.803761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761623.803769:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761623.803774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1601 00000100:00080000:3.0:1789761623.803774:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761623.803777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761623.803778:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507522432:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761623.803781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761623.803785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761623.803794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507523072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761623.803800:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761623.803823:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761623.803837:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761623/real 1789761623] req@ffff8d25d25b9f80 x1876699507522560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761623.803856:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507522560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761623.803873:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761623.803877:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761623.803881:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761623.803888:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761623.803910:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761623.803922:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761623/real 1789761623] req@ffff8d25d25bb800 x1876699507522688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761623.803935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507522688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761623.803942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761623.803946:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761623.803949:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761623.803955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761623.803978:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761623.803990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761623/real 1789761623] req@ffff8d25d25ba680 x1876699507522816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761623.804004:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507522816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761623.804020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761623.804024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761623.804028:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507522816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761623.804035:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507522944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761623.804058:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761623.804139:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761623/real 1789761623] req@ffff8d25d25b9180 x1876699507522944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761623.804154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507522944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761623.804163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761623.804258:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761623.804263:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507522944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761623.804282:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507523072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761623.804372:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761623.804420:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761623/real 1789761623] req@ffff8d25d25b8a80 x1876699507523072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761623.804434:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507523072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761648 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761623.804443:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761623.804447:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761623.804461:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507523072:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761623.867489:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761624.891559:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761625.915514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761626.939511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761627.963515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761628.923532:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761628.923548:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761628.923554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761628.923567:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1606 00000100:00080000:1.0:1789761628.923579:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761628.923588:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761628.923637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507523200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761628.923670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761628.923675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761628.923677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761628.923684:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1606 00000100:00080000:1.0:1789761628.923689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761628.923694:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761628.923704:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507523200:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789761628.923707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761628.923726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507523328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761628.923738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761628.923743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761628.923745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761628.923749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1606 00000100:00080000:1.0:1789761628.923754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761628.923759:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761628.923765:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761628.923779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507523456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761628.923781:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761628.923801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761628.923805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761628.923807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761628.923813:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1606 00000100:00080000:1.0:1789761628.923818:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761628.923822:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761628.923828:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761628.923839:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761628/real 1789761628] req@ffff8d25c8168380 x1876699507523200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761628.923845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507523584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761628.923862:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761628.923864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507523200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761628.923869:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761628.923873:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761628.923882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1606 00000100:00080000:3.0:1789761628.923882:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761628.923890:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789761628.923891:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761628.923895:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507523200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761628.923901:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761628.923908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761628.923923:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507523328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761628.923933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507523712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761628.923943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761628.923949:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761628.923955:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761628.923957:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761628.923963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761628/real 1789761628] req@ffff8d25c8168000 x1876699507523328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761628.923965:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1606 00000100:00080000:1.0:1789761628.923973:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761628.923981:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761628.923989:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761628.923991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507523328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761628.924004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789761628.924005:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507523840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761628.924013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761628.924021:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507523328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761628.924033:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507523456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761628.924054:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761628.924063:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761628/real 1789761628] req@ffff8d25c8169500 x1876699507523456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761628.924072:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507523456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761628.924077:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761628.924080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761628.924082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507523456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761628.924085:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507523584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761628.924097:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761628.924103:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761628/real 1789761628] req@ffff8d25c816aa00 x1876699507523584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761628.924110:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507523584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761628.924116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761628.924118:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761628.924119:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507523584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761628.924132:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507523712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761628.924146:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761628.924157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761628/real 1789761628] req@ffff8d25c816a680 x1876699507523712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761628.924170:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507523712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761628.924179:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761628.924182:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761628.924186:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507523712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761628.924192:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507523840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761628.924214:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761628.924226:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761628/real 1789761628] req@ffff8d25c8169f80 x1876699507523840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761628.924240:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507523840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761653 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761628.924248:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761628.924252:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761628.924256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507523840:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761628.987549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761630.011513:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761631.035565:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761632.059581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761633.083590:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761634.043462:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761634.043473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761634.043478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761634.043488:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1611 00000100:00080000:1.0:1789761634.043499:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761634.043507:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761634.043545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507523968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761634.043570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761634.043573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761634.043578:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761634.043582:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1611 00000100:00080000:1.0:1789761634.043586:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761634.043592:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761634.043603:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761634.043613:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507523968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761634.043621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507524096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761634.043635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761634.043640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761634.043643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761634.043648:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1611 00000100:00080000:1.0:1789761634.043653:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761634.043658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761634.043664:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761634.043674:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761634.043681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507524224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761634.043693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761634.043697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761634.043700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761634.043704:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1611 00000100:00080000:1.0:1789761634.043708:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761634.043713:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761634.043718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761634.043729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507524352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761634.043735:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffff8d25c8168e00 x1876699507523968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761634.043737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761634.043744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761634.043747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761634.043754:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1611 00000100:00080000:1.0:1789761634.043759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761634.043765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761634.043772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761634.043780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507524480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761634.043785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507523968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761634.043797:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761634.043798:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761634.043803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761634.043807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761634.043807:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789761634.043816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1611 00000100:00100000:3.0:1789761634.043817:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507523968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761634.043820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761634.043825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761634.043829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761634.043836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507524608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761634.043842:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761634.043866:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761634.043876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffff8d25c8168700 x1876699507524096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.043885:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507524096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.043891:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.043901:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761634.043903:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761634.043907:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761634.043920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761634.043927:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffff8d25c8169f80 x1876699507524224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.043935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507524224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.043944:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.043947:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761634.043949:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761634.043955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761634.043974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761634.043986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffff8d25c816a680 x1876699507524352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.044000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507524352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.044009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.044012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761634.044014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761634.044018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507524480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761634.044036:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761634.044049:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffff8d25c816aa00 x1876699507524480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.044061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507524480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.044070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.044073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761634.044078:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507524480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761634.044094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507524608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761634.044110:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761634.044118:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761634/real 1789761634] req@ffff8d25c8169500 x1876699507524608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761634.044126:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507524608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761659 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761634.044131:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761634.044133:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761634.044135:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507524608:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761634.107507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761635.131488:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761636.155521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761637.179549:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761638.203519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761639.163471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761639.163480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761639.163484:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761639.163492:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1616 00000100:00080000:1.0:1789761639.163500:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761639.163506:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761639.163536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507524736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761639.163558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761639.163560:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761639.163561:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761639.163565:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1616 00000100:00080000:1.0:1789761639.163567:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761639.163570:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761639.163578:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761639.163587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507524864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761639.163594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761639.163597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761639.163598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761639.163601:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1616 00000100:00080000:1.0:1789761639.163602:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761639.163605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761639.163608:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761639.163611:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507524736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761639.163617:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507524992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761639.163625:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761639.163628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761639.163630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761639.163632:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1616 00000100:00080000:1.0:1789761639.163634:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761639.163636:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761639.163639:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761639.163648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507525120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761639.163659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761639.163661:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761639.163662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761639.163668:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1616 00000800:00000100:3.0:1789761639.163670:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761639.163672:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761639.163676:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761639.163679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761639.163688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507525248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761639.163694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761639.163697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761639.163698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761639.163700:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1616 00000100:00080000:1.0:1789761639.163702:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761639.163705:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761639.163708:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761639.163720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507525376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761639.163720:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffff8d25c8168000 x1876699507524736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.163752:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507524736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.163761:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.163764:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761639.163767:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507524736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761639.163773:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761639.163792:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761639.163801:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffff8d25c8168380 x1876699507524864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.163810:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507524864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.163815:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.163817:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761639.163820:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761639.163836:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761639.163848:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761639.163854:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffff8d25c816b480 x1876699507524992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.163862:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507524992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.163867:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.163869:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761639.163871:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507524992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761639.163877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761639.163895:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761639.163906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffff8d25c816a300 x1876699507525120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.163920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507525120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.163929:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.163932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761639.163934:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761639.163938:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507525248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761639.163953:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761639.163962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffff8d25c816ad80 x1876699507525248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.163969:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507525248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.163974:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.163976:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761639.163978:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507525248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761639.163982:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507525376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761639.163992:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761639.164002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761639/real 1789761639] req@ffff8d25c8168a80 x1876699507525376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761639.164014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507525376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761664 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761639.164028:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761639.164031:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761639.164035:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507525376:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761639.227523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761640.251533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761641.275607:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761642.299526:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761643.323556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761644.283493:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761644.283502:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761644.283505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761644.283516:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1621 00000100:00080000:0.0:1789761644.283525:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761644.283532:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761644.283572:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507525504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761644.283595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761644.283598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761644.283600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761644.283604:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1621 00000100:00080000:0.0:1789761644.283608:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761644.283612:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761644.283624:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761644.283627:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507525504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761644.283637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507525632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761644.283646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761644.283649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761644.283650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761644.283653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1621 00000100:00080000:0.0:1789761644.283656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761644.283658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761644.283663:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761644.283670:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507525760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761644.283677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761644.283679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761644.283680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761644.283683:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1621 00000100:00080000:0.0:1789761644.283686:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789761644.283690:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761644.283691:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761644.283697:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761644.283705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507525888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761644.283713:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761644.283715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761644.283716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761644.283719:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1621 00000100:00080000:0.0:1789761644.283721:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761644.283723:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761644.283726:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761644.283741:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507526016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761644.283742:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffff8d25d25bbb80 x1876699507525504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761644.283762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761644.283768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761644.283771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761644.283779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1621 00000100:00080000:0.0:1789761644.283782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761644.283785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761644.283788:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761644.283790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507525504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761644.283797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507526144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761644.283802:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.283808:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761644.283812:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507525504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761644.283832:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761644.283850:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761644.283859:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffff8d25d25b8000 x1876699507525632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.283867:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507525632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.283873:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.283875:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761644.283877:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761644.283881:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761644.283891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761644.283897:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffff8d25d25ba300 x1876699507525760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.283905:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507525760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.283914:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.283916:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761644.283918:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761644.283921:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789761644.283931:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761644.283937:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffff8d25d25b8a80 x1876699507525888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.283944:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507525888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.283950:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.283951:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761644.283953:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507525888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789761644.283956:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761644.283966:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761644.283972:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffff8d25d25b9180 x1876699507526016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.283980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507526016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.283984:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.283986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761644.283988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761644.283991:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761644.284001:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761644.284006:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761644/real 1789761644] req@ffff8d25d25ba680 x1876699507526144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761644.284014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507526144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761669 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761644.284022:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761644.284025:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761644.284028:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526144:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761644.347488:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761645.371537:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761646.395480:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761647.419545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761648.443512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761649.403534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761649.403545:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761649.403550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761649.403561:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1627 00000100:00080000:3.0:1789761649.403572:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761649.403579:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761649.403631:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507526272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761649.403662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761649.403665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761649.403667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761649.403671:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1627 00000100:00080000:3.0:1789761649.403675:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761649.403678:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761649.403688:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761649.403701:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507526400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761649.403711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761649.403714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761649.403716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761649.403717:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507526272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761649.403721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1627 00000100:00080000:3.0:1789761649.403728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761649.403731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761649.403734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761649.403743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507526528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761649.403750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761649.403754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761649.403755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761649.403758:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1627 00000100:00080000:3.0:1789761649.403760:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761649.403763:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761649.403766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761649.403774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507526656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761649.403783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761649.403787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761649.403789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761649.403794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1627 00000800:00000100:1.0:1789761649.403798:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761649.403802:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761649.403810:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761649.403817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761649.403831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507526784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761649.403842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761649.403847:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761649/real 1789761649] req@ffff8d25d1b0b100 x1876699507526272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761649.403849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761649.403854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789761649.403868:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507526272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761649.403876:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1627 00000100:00080000:1.0:1789761649.403881:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761649.403884:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761649.403887:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761649.403895:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507526272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761649.403895:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761649.403906:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761649.403922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507526912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761649.403926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507526400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761649.403963:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761649.403983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761649/real 1789761649] req@ffff8d25d1b0b480 x1876699507526400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761649.403998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507526400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761649.404007:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761649.404011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761649.404015:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507526400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761649.404020:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507526528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761649.404037:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761649.404046:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761649/real 1789761649] req@ffff8d25d1b09500 x1876699507526528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761649.404054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507526528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761649.404060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761649.404062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761649.404064:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507526528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761649.404068:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507526656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789761649.404079:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761649.404085:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761649/real 1789761649] req@ffff8d25d1b08000 x1876699507526656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761649.404092:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507526656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761649.404098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761649.404100:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761649.404103:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507526656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789761649.404106:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761649.404116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761649.404122:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761649/real 1789761649] req@ffff8d25d1b08380 x1876699507526784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761649.404129:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507526784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761649.404135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761649.404136:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761649.404138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761649.404153:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761649.404164:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761649.404170:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761649/real 1789761649] req@ffff8d25d1b0a300 x1876699507526912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761649.404177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507526912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761674 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761649.404181:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761649.404183:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761649.404186:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507526912:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761649.467604:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761650.491543:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761651.515531:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761652.539524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761653.563506:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761654.523470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761654.523478:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761654.523482:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761654.523491:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1632 00000100:00080000:3.0:1789761654.523499:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761654.523505:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761654.523537:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507527040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761654.523560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761654.523562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761654.523564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761654.523567:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1632 00000100:00080000:3.0:1789761654.523569:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761654.523572:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761654.523579:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761654.523588:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507527168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761654.523596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789761654.523601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507527040:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761654.523601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761654.523605:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761654.523613:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1632 00000100:00080000:3.0:1789761654.523615:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761654.523618:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761654.523620:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761654.523630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507527296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761654.523636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761654.523639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761654.523640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761654.523644:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1632 00000100:00080000:3.0:1789761654.523646:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761654.523649:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761654.523652:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761654.523660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507527424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761654.523667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789761654.523672:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761654.523673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761654.523676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761654.523680:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1632 00000100:00080000:3.0:1789761654.523683:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761654.523685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761654.523689:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761654.523697:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507527552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761654.523703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761654.523705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761654.523706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761654.523709:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1632 00000100:00080000:3.0:1789761654.523712:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761654.523715:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761654.523718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761654.523725:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761654/real 1789761654] req@ffff8d25d1b0a300 x1876699507527040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761654.523728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507527680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789761654.523771:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507527040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761654.523784:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761654.523789:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761654.523793:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507527040:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761654.523817:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761654.523835:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761654.523848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761654/real 1789761654] req@ffff8d25d1b08380 x1876699507527168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761654.523861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507527168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761654.523871:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761654.523874:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761654.523878:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761654.523885:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761654.523907:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761654.523921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761654/real 1789761654] req@ffff8d25d1b08000 x1876699507527296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761654.523935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507527296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761654.523944:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761654.523948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761654.523959:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761654.523966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789761654.523986:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761654.523997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761654/real 1789761654] req@ffff8d25d1b09500 x1876699507527424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761654.524010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507527424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761654.524018:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761654.524021:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761654.524024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789761654.524028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507527552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761654.524044:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761654.524055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761654/real 1789761654] req@ffff8d25d1b0b480 x1876699507527552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761654.524067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507527552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761654.524085:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761654.524089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761654.524093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507527552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761654.524100:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507527680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789761654.524121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761654.524130:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761654/real 1789761654] req@ffff8d25d1b0b100 x1876699507527680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761654.524138:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507527680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761679 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761654.524143:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761654.524145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789761654.524147:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507527680:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761654.587486:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761655.611524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761656.635501:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761657.659482:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761658.683582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761659.643487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761659.643495:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761659.643499:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761659.643508:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1637 00000100:00080000:2.0:1789761659.643516:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761659.643522:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761659.643554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507527808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761659.643586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761659.643589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761659.643592:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761659.643601:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1637 00000100:00080000:2.0:1789761659.643606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761659.643613:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761659.643624:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761659.643631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507527808:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761659.643648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507527936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761659.643677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761659.643681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761659.643683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761659.643691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1637 00000100:00080000:2.0:1789761659.643707:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761659.643712:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761659.643718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761659.643727:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761659.643732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507528064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761659.643742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761659.643746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761659.643747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761659.643751:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1637 00000100:00080000:2.0:1789761659.643755:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761659.643758:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761659.643761:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761659.643771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507528192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761659.643778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761659.643781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761659.643782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761659.643784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1637 00000100:00080000:2.0:1789761659.643785:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761659.643788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761659.643790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761659.643795:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507528320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761659.643801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761659.643802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761659.643804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761659.643807:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1637 00000100:00080000:2.0:1789761659.643809:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761659.643811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761659.643814:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761659.643821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507528448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761659.643837:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761659/real 1789761659] req@ffff8d25c8020380 x1876699507527808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761659.643858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507527808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761659.643870:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761659.643876:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761659.643881:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507527808:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761659.643912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761659.643944:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761659.643961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761659/real 1789761659] req@ffff8d25c8022a00 x1876699507527936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761659.643976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507527936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761659.643986:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761659.643990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761659.644102:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507527936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761659.644110:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761659.644216:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761659.644244:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761659/real 1789761659] req@ffff8d25c8022680 x1876699507528064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761659.644260:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507528064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761659.644271:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761659.644275:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761659.644279:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761659.644287:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761659.644311:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761659.644322:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761659/real 1789761659] req@ffff8d25c8021180 x1876699507528192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761659.644331:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507528192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761659.644338:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761659.644340:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761659.644343:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761659.644347:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507528320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761659.644362:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761659.644369:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761659/real 1789761659] req@ffff8d25c8020a80 x1876699507528320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761659.644422:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507528320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761659.644434:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761659.644439:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761659.644444:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507528320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761659.644453:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507528448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761659.644477:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761659.644490:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761659/real 1789761659] req@ffff8d25c8023800 x1876699507528448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761659.644502:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507528448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761684 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761659.644508:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761659.644510:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761659.644512:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507528448:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761659.707514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761660.731527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761661.755498:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761662.779512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761663.803522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761664.763483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761664.763494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761664.763498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761664.763509:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1642 00000100:00080000:2.0:1789761664.763520:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761664.763529:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761664.763569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507528576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761664.763594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761664.763598:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761664.763599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761664.763603:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1642 00000100:00080000:2.0:1789761664.763607:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761664.763611:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761664.763620:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761664.763636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507528704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761664.763647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761664.763650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761664.763652:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761664.763652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507528576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761664.763662:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1642 00000100:00080000:2.0:1789761664.763669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761664.763672:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761664.763675:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761664.763685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507528832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761664.763692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761664.763695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761664.763697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761664.763700:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1642 00000100:00080000:2.0:1789761664.763704:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761664.763707:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761664.763709:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761664.763718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507528960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761664.763725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761664.763728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761664.763730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761664.763734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1642 00000100:00080000:2.0:1789761664.763739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761664.763743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761664.763749:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761664.763762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507529088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761664.763777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761664.763780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761664.763781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761664.763785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1642 00000100:00080000:2.0:1789761664.763790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761664.763795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761664.763801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761664.763817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507529216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761664.763825:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761664.763873:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761664.763905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761664.763929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761664.763958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529216:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761664.827605:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761665.851512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761666.875558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789761667.835635:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789761667.835678:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761667.835694:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761667.835733:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761667.835745:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761667.835756:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761667.835768:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761667.835777:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761664/real 1789761667] req@ffff8d25c8020a80 x1876699507528704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1789761667.835779:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761667.835809:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507528704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761667.835822:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761667.835828:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761667.835833:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761667.835843:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761664/real 1789761667] req@ffff8d25c8021180 x1876699507528832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761667.835857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507528832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761667.835867:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761667.835870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761667.835874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761667.835881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761664/real 1789761667] req@ffff8d25c8022680 x1876699507528960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761667.835902:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507528960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761667.835911:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761667.835915:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761667.835919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507528960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761667.835926:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761664/real 1789761667] req@ffff8d25c8022a00 x1876699507529088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761667.835942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507529088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761667.835951:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761667.835954:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761667.835957:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761667.835965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761664/real 1789761667] req@ffff8d25c8023800 x1876699507528576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761667.835979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507528576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761667.835987:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761667.835997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761667.836000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507528576:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761667.836031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761664/real 1789761667] req@ffff8d25c8020380 x1876699507529216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761667.836046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507529216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761689 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761667.836054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761667.836069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761667.836073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529216:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761667.899547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761668.923501:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761669.883477:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761669.883491:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761669.883497:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761669.883512:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1647 00000100:00080000:2.0:1789761669.883528:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761669.883539:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761669.883588:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507529344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761669.883621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761669.883625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761669.883627:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761669.883633:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1647 00000100:00080000:2.0:1789761669.883637:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761669.883653:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761669.883665:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761669.883670:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507529344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761669.883683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507529472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761669.883701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761669.883706:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761669.883708:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761669.883713:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1647 00000100:00080000:2.0:1789761669.883717:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761669.883721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761669.883728:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761669.883741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507529600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761669.883753:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761669.883759:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761669.883760:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761669.883766:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761669.883772:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1647 00000100:00080000:2.0:1789761669.883777:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761669.883782:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761669.883789:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761669.883804:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507529728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761669.883815:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761669.883819:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761669.883821:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761669.883824:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1647 00000100:00080000:2.0:1789761669.883828:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761669.883833:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761669.883840:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761669.883842:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761669/real 1789761669] req@ffff8d25c8023100 x1876699507529344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761669.883855:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507529856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761669.883866:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761669.883873:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761669.883873:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507529344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761669.883877:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761669.883887:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1647 00000100:00080000:2.0:1789761669.883894:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761669.883897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761669.883899:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761669.883906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789761669.883908:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761669.883914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507529344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761669.883926:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507529984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761669.883952:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507529472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761669.883985:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761669.884000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761669/real 1789761669] req@ffff8d25c8023b80 x1876699507529472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761669.884013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507529472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761669.884024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761669.884028:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761669.884032:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507529472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761669.884039:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507529600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761669.884076:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761669.884089:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761669/real 1789761669] req@ffff8d25c8020000 x1876699507529600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761669.884102:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507529600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761669.884110:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761669.884114:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761669.884117:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507529600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761669.884123:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507529728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761669.884144:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761669.884157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761669/real 1789761669] req@ffff8d25c8020700 x1876699507529728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761669.884168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507529728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761669.884176:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761669.884180:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761669.884184:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507529728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761669.884189:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761669.884209:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761669.884301:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761669/real 1789761669] req@ffff8d25c8022d80 x1876699507529856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761669.884314:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507529856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761669.884323:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761669.884326:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761669.884329:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761669.884336:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761669.884360:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761669.884373:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761669/real 1789761669] req@ffff8d25c8022300 x1876699507529984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761669.884403:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507529984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761694 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761669.884409:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761669.884412:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761669.884416:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507529984:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761669.947528:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761670.971543:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761671.995508:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761673.019611:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761674.043512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761675.003495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761675.003505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761675.003510:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761675.003521:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1652 00000100:00080000:1.0:1789761675.003532:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761675.003540:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761675.003572:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507530112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761675.003595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761675.003598:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761675.003599:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761675.003603:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1652 00000100:00080000:1.0:1789761675.003605:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761675.003608:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761675.003616:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761675.003626:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507530240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761675.003635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761675.003635:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507530112:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761675.003643:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761675.003646:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761675.003649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1652 00000100:00080000:1.0:1789761675.003651:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761675.003653:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761675.003656:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761675.003665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507530368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761675.003671:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761675.003674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761675.003675:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761675.003678:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1652 00000100:00080000:1.0:1789761675.003680:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761675.003682:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761675.003686:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761675.003694:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507530496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761675.003701:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761675.003706:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761675.003707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761675.003711:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761675.003715:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1652 00000100:00080000:1.0:1789761675.003718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761675.003721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761675.003725:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761675.003733:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507530624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761675.003739:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761675.003741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761675.003743:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761675.003748:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1652 00000100:00080000:1.0:1789761675.003750:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761675.003751:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffff8d25c8169880 x1876699507530112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761675.003754:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761675.003760:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761675.003771:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507530752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761675.003785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507530112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761675.003798:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761675.003801:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761675.003805:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507530112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761675.003823:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507530240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761675.003842:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761675.003852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffff8d25c8169c00 x1876699507530240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761675.003861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507530240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761675.003866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761675.003869:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761675.003871:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507530240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761675.003874:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507530368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761675.003885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761675.003891:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffff8d25c8168a80 x1876699507530368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761675.003900:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507530368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761675.003905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761675.003907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761675.003909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507530368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761675.003912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507530496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761675.003923:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761675.003929:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffff8d25c816ad80 x1876699507530496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761675.003936:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507530496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761675.003941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761675.003943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761675.003945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507530496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761675.003948:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507530624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761675.003958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761675.003964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffff8d25c816a300 x1876699507530624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761675.003971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507530624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761675.003981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761675.003983:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761675.003985:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507530624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761675.003988:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507530752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761675.003997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761675.004020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761675/real 1789761675] req@ffff8d25c816b480 x1876699507530752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761675.004028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507530752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761700 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761675.004033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761675.004035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761675.004037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507530752:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761675.067512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761676.091563:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761677.115519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761678.139565:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761679.163595:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761680.123495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761680.123504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761680.123508:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761680.123518:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1657 00000100:00080000:1.0:1789761680.123530:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761680.123539:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761680.123577:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507530880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761680.123604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761680.123606:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761680.123608:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761680.123611:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1657 00000100:00080000:1.0:1789761680.123614:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761680.123618:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761680.123627:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761680.123638:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507531008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761680.123646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761680.123650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761680.123653:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761680.123661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507530880:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761680.123669:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1657 00000100:00080000:1.0:1789761680.123676:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761680.123679:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761680.123683:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761680.123695:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507531136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761680.123706:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761680.123709:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761680.123711:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761680.123714:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1657 00000100:00080000:1.0:1789761680.123717:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761680.123719:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761680.123723:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761680.123733:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507531264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761680.123743:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761680.123747:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761680.123749:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761680.123754:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1657 00000100:00080000:1.0:1789761680.123759:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761680.123763:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761680.123768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761680.123772:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761680.123787:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507531392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761680.123799:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761680.123803:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761680.123806:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761680.123810:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1657 00000100:00080000:1.0:1789761680.123816:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761680.123821:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761680.123826:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761680.123848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507531520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761680.123849:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffff8d25c8168380 x1876699507530880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761680.123909:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507530880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761680.123926:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761680.123929:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761680.123933:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507530880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761680.123957:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761680.123997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761680.124015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffff8d25c8168000 x1876699507531008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761680.124024:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507531008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761680.124030:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761680.124033:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761680.124035:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761680.124043:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761680.124065:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761680.124079:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffff8d25c8169500 x1876699507531136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761680.124092:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507531136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761680.124102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761680.124105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761680.124109:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761680.124115:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761680.124133:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761680.124142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffff8d25c816aa00 x1876699507531264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761680.124150:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507531264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761680.124156:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761680.124158:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761680.124161:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761680.124164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507531392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761680.124175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761680.124182:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffff8d25c816a680 x1876699507531392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761680.124189:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507531392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761680.124195:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761680.124196:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761680.124198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507531392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761680.124202:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507531520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761680.124213:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761680.124219:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761680/real 1789761680] req@ffff8d25c8169f80 x1876699507531520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761680.124227:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507531520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761705 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761680.124232:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761680.124234:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761680.124236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507531520:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761680.187562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761681.211495:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761682.235562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761683.259527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761684.283558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761685.243488:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761685.243498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761685.243503:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761685.243521:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1662 00000100:00080000:0.0:1789761685.243532:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761685.243539:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761685.243587:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507531648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761685.243621:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761685.243625:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761685.243627:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761685.243631:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1662 00000100:00080000:0.0:1789761685.243635:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761685.243639:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761685.243651:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761685.243663:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507531776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761685.243676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761685.243678:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507531648:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761685.243681:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761685.243687:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761685.243698:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1662 00000100:00080000:0.0:1789761685.243700:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761685.243703:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761685.243706:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761685.243715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507531904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761685.243721:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761685.243725:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761685.243728:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761685.243733:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1662 00000100:00080000:0.0:1789761685.243737:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761685.243742:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761685.243749:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761685.243764:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507532032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761685.243766:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761685.243784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761685.243788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761685.243790:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761685.243794:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1662 00000100:00080000:0.0:1789761685.243799:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761685.243804:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761685.243810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761685.243811:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761685] req@ffff8d25d25bb800 x1876699507531648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761685.243825:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507532160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761685.243839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761685.243843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761685.243845:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761685.243852:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1662 00000100:00100000:3.0:1789761685.243852:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507531648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761685.243860:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761685.243870:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761685.243880:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761685.243894:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789761685.243900:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507532288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761685.243906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761685.243920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507531648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761685.243967:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761685.243988:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761685.243999:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761685] req@ffff8d25d25b9f80 x1876699507531776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761685.244013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507531776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761685.244019:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761685.244022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761685.244024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761685.244028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761685.244040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761685.244047:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761685] req@ffff8d25d25bad80 x1876699507531904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761685.244055:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507531904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761685.244060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761685.244062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761685.244064:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507531904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761685.244068:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761685.244079:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761685.244085:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761685] req@ffff8d25d25b8380 x1876699507532032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761685.244092:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507532032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761685.244102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761685.244104:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761685.244107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761685.244110:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507532160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761685.244120:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761685.244126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761685] req@ffff8d25d25b8700 x1876699507532160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761685.244133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507532160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761685.244138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761685.244140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761685.244142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507532160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761685.244146:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507532288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761685.244156:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761685.244162:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761685/real 1789761685] req@ffff8d25d25bb100 x1876699507532288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761685.244168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507532288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761710 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761685.244173:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761685.244175:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761685.244178:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507532288:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761685.307505:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761686.331550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761687.355532:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761688.379545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761689.403511:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761690.363493:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761690.363506:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761690.363511:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761690.363529:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1668 00000100:00080000:0.0:1789761690.363540:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761690.363547:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761690.363576:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507532416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761690.363596:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761690.363599:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761690.363600:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761690.363604:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1668 00000100:00080000:0.0:1789761690.363606:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761690.363608:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761690.363616:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761690.363625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507532544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761690.363634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761690.363638:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761690.363640:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507532416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761690.363641:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761690.363651:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1668 00000100:00080000:0.0:1789761690.363655:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761690.363659:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761690.363664:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761690.363677:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507532672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761690.363695:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761690.363699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761690.363706:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761690.363708:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761690.363716:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1668 00000100:00080000:0.0:1789761690.363722:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761690.363728:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761690.363734:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761690.363745:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffff8d25d25b9c00 x1876699507532416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761690.363752:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507532800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761690.363773:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761690.363776:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761690.363779:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761690.363780:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507532416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761690.363784:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1668 00000100:00080000:0.0:1789761690.363789:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761690.363797:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761690.363805:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789761690.363806:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761690.363815:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761690.363821:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507532416:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761690.363826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507532928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761690.363839:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761690.363843:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761690.363847:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532544:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761690.363848:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761690.363858:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1668 00000100:00080000:0.0:1789761690.363864:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761690.363869:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761690.363876:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761690.363876:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761690.363903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffff8d25d25b8e00 x1876699507532544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761690.363909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507533056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761690.363927:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507532544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.363935:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.363938:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761690.363942:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761690.363950:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761690.363969:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761690.363980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffff8d25d25bb480 x1876699507532672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761690.363989:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507532672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.363994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.363997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761690.363999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761690.364002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761690.364018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761690.364029:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffff8d25d25bb100 x1876699507532800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761690.364057:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507532800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.364066:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.364069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761690.364071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507532800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761690.364075:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507532928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761690.364092:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761690.364100:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffff8d25d25b8700 x1876699507532928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761690.364108:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507532928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.364114:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.364116:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761690.364118:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507532928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761690.364130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507533056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761690.364141:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761690.364147:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761690/real 1789761690] req@ffff8d25d25b8380 x1876699507533056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761690.364155:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507533056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761715 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761690.364160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761690.364161:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761690.364163:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507533056:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761690.427544:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761691.451518:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761692.475504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761693.499499:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761694.523561:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761695.483483:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761695.483500:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761695.483506:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761695.483520:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1673 00000100:00080000:2.0:1789761695.483534:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761695.483544:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761695.483585:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507533184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761695.483613:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761695.483616:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761695.483618:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761695.483623:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1673 00000100:00080000:2.0:1789761695.483626:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761695.483631:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761695.483641:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761695.483647:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507533184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761695.483656:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507533312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761695.483666:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761695.483670:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761695.483672:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761695.483676:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1673 00000100:00080000:2.0:1789761695.483679:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761695.483682:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761695.483688:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761695.483697:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507533440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761695.483704:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761695.483707:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761695.483709:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761695.483711:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1673 00000100:00080000:2.0:1789761695.483714:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761695.483716:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761695.483719:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789761695.483720:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761695.483732:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507533568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761695.483741:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761695.483744:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761695.483745:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761695.483748:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1673 00000100:00080000:2.0:1789761695.483750:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761695.483753:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761695.483756:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761695.483765:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507533696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761695.483770:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761695/real 1789761695] req@ffff8d25c8021880 x1876699507533184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761695.483776:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761695.483784:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761695.483788:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761695.483793:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1673 00000100:00080000:2.0:1789761695.483796:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761695.483801:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761695.483802:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507533184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761695.483807:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761695.483820:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761695.483820:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507533824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761695.483832:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761695.483841:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507533184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761695.483884:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507533312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761695.483923:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761695.483939:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761695/real 1789761695] req@ffff8d25c8021c00 x1876699507533312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761695.483954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507533312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761695.483964:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761695.483968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761695.483973:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507533312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761695.483981:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507533440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761695.484006:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761695.484021:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761695/real 1789761695] req@ffff8d25c8021500 x1876699507533440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761695.484035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507533440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761695.484045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761695.484050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761695.484055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507533440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761695.484064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507533568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761695.484092:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761695.484113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761695/real 1789761695] req@ffff8d25c8020e00 x1876699507533568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761695.484130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507533568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761695.484141:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761695.484145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761695.484149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507533568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761695.484155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507533696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761695.484174:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761695.484184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761695/real 1789761695] req@ffff8d25c8023480 x1876699507533696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761695.484192:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507533696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761695.484198:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761695.484200:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761695.484202:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507533696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761695.484206:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507533824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761695.484218:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761695.484229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761695/real 1789761695] req@ffff8d25c8022300 x1876699507533824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761695.484241:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507533824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761720 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761695.484250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761695.484253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761695.484257:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507533824:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761695.547537:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761696.571529:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761697.595551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761698.619507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761699.643526:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761700.603490:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761700.603498:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761700.603501:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761700.603509:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1678 00000100:00080000:0.0:1789761700.603516:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761700.603522:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761700.603557:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507533952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761700.603586:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761700.603590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761700.603593:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761700.603600:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1678 00000100:00080000:0.0:1789761700.603605:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761700.603610:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761700.603617:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507533952:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761700.603620:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761700.603637:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507534080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761700.603646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761700.603649:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761700.603651:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761700.603654:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1678 00000100:00080000:0.0:1789761700.603657:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761700.603660:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761700.603663:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761700.603672:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507534208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761700.603680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761700.603684:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761700.603685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761700.603690:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761700.603694:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1678 00000100:00080000:0.0:1789761700.603697:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761700.603699:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761700.603703:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761700.603711:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507534336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761700.603718:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761700.603721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761700.603722:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761700.603725:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1678 00000100:00080000:0.0:1789761700.603727:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761700.603732:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761700.603734:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761700/real 1789761700] req@ffff8d25d25bad80 x1876699507533952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761700.603735:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761700.603747:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507534464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761700.603756:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761700.603758:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761700.603760:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761700.603766:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1678 00000100:00100000:3.0:1789761700.603768:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507533952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761700.603771:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761700.603779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761700.603789:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761700.603790:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761700.603798:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761700.603801:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507533952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761700.603808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507534592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761700.603843:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761700.603865:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761700.603876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761700/real 1789761700] req@ffff8d25d25b9f80 x1876699507534080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761700.603890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507534080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761700.603899:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761700.603903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761700.603906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761700.603913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761700.603935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761700.603949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761700/real 1789761700] req@ffff8d25d25bb800 x1876699507534208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761700.603969:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507534208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761700.603977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761700.603980:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761700.603982:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761700.603986:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761700.604003:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761700.604012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761700/real 1789761700] req@ffff8d25d25ba680 x1876699507534336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761700.604019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507534336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761700.604025:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761700.604027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761700.604029:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761700.604032:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507534464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761700.604042:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761700.604048:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761700/real 1789761700] req@ffff8d25d25b9180 x1876699507534464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761700.604055:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507534464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761700.604069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761700.604073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761700.604076:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507534464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761700.604082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507534592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761700.604103:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761700.604116:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761700/real 1789761700] req@ffff8d25d25b8a80 x1876699507534592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761700.604129:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507534592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761725 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761700.604137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761700.604140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761700.604143:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507534592:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761700.667517:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761701.691537:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761702.715505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761703.739529:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761704.763495:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761705.723480:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761705.723489:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761705.723492:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761705.723501:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1683 00000100:00080000:2.0:1789761705.723509:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761705.723515:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761705.723545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507534720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761705.723565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761705.723568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761705.723569:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761705.723572:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1683 00000100:00080000:2.0:1789761705.723574:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761705.723577:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761705.723584:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761705.723594:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507534848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761705.723603:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761705.723604:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507534720:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761705.723608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761705.723613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761705.723617:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1683 00000100:00080000:2.0:1789761705.723619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761705.723621:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761705.723624:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761705.723632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507534976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761705.723639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761705.723641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761705.723643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761705.723645:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1683 00000100:00080000:2.0:1789761705.723650:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761705.723653:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761705.723656:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761705.723662:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761705.723666:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507535104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761705.723675:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761705.723678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761705.723679:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761705.723682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1683 00000100:00080000:2.0:1789761705.723684:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761705.723686:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761705.723689:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761705.723698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507535232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761705.723703:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761705/real 1789761705] req@ffff8d25c8022d80 x1876699507534720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761705.723707:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761705.723714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761705.723717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761705.723723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1683 00000100:00080000:2.0:1789761705.723726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761705.723731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761705.723733:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507534720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761705.723734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761705.723745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507535360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761705.723791:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761705.723796:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761705.723800:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507534720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761705.723820:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761705.723840:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761705.723850:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761705/real 1789761705] req@ffff8d25c8020700 x1876699507534848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761705.723859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507534848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761705.723865:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761705.723867:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761705.723869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761705.723873:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761705.723884:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761705.723890:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761705/real 1789761705] req@ffff8d25c8020000 x1876699507534976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761705.723898:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507534976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761705.723903:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761705.723905:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761705.723907:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507534976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761705.723911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761705.723922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761705.723928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761705/real 1789761705] req@ffff8d25c8023b80 x1876699507535104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761705.723936:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507535104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761705.723941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761705.723943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761705.723945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761705.723948:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507535232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761705.723958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761705.723965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761705/real 1789761705] req@ffff8d25c8023100 x1876699507535232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761705.723972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507535232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761705.723977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761705.723979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761705.723981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507535232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761705.723985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507535360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761705.723994:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761705.724000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761705/real 1789761705] req@ffff8d25c8020380 x1876699507535360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761705.724008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507535360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761730 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761705.724013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761705.724015:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761705.724016:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507535360:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761705.787515:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761706.811517:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761707.835577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761708.859490:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761709.883558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761710.843524:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761710.843534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761710.843539:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761710.843549:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1688 00000100:00080000:3.0:1789761710.843560:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761710.843567:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761710.843614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507535488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761710.843650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761710.843655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761710.843656:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761710.843661:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1688 00000100:00080000:3.0:1789761710.843664:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761710.843669:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761710.843680:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761710.843694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507535616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761710.843706:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507535488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761710.843708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761710.843720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761710.843725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761710.843729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1688 00000100:00080000:3.0:1789761710.843747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761710.843752:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761710.843757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761710.843775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507535744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761710.843783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761710.843785:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761710.843787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761710.843790:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1688 00000100:00080000:3.0:1789761710.843794:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761710.843796:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761710.843804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789761710.843810:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761710.843814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507535872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761710.843823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761710.843826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761710.843828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761710.843832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1688 00000100:00080000:3.0:1789761710.843835:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761710.843839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761710.843843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761710.843859:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761710/real 1789761710] req@ffff8d25d1b09c00 x1876699507535488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761710.843860:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507536000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761710.843925:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761710.843929:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507535488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761710.843932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761710.843937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761710.843951:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1688 00000100:00080000:0.0:1789761710.843954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761710.843955:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761710.843963:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761710.843969:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789761710.843973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761710.843980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507535488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761710.843988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507536128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761710.844015:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761710.844039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761710.844049:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761710/real 1789761710] req@ffff8d25d1b0ad80 x1876699507535616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761710.844059:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507535616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761710.844065:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761710.844067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761710.844069:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761710.844073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761710.844084:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761710.844092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761710/real 1789761710] req@ffff8d25d1b09180 x1876699507535744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761710.844100:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507535744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761710.844105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761710.844107:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761710.844108:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761710.844112:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761710.844123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761710.844129:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761710/real 1789761710] req@ffff8d25d1b0aa00 x1876699507535872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761710.844137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507535872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761710.844142:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761710.844144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761710.844146:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507535872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761710.844149:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761710.844159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761710.844166:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761710/real 1789761710] req@ffff8d25d1b08a80 x1876699507536000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761710.844173:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507536000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761710.844179:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761710.844181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761710.844183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761710.844186:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761710.844196:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761710.844202:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761710/real 1789761710] req@ffff8d25d1b0bb80 x1876699507536128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761710.844209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507536128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761735 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761710.844213:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761710.844215:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761710.844217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536128:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761710.907501:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761711.931567:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761712.955502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761713.979506:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761715.003540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761715.963464:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761715.963472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761715.963476:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761715.963485:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1693 00000100:00080000:3.0:1789761715.963494:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761715.963499:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761715.963531:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507536256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761715.963556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761715.963558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761715.963560:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761715.963563:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1693 00000100:00080000:3.0:1789761715.963565:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761715.963567:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761715.963574:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761715.963583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507536384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761715.963590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761715.963593:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761715.963594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761715.963597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1693 00000100:00080000:3.0:1789761715.963599:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761715.963603:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761715.963605:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507536256:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789761715.963606:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761715.963617:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507536512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761715.963624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761715.963627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761715.963628:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761715.963631:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1693 00000100:00080000:3.0:1789761715.963633:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761715.963635:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761715.963639:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761715.963646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507536640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761715.963653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761715.963655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761715.963657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761715.963660:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1693 00000100:00080000:3.0:1789761715.963664:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761715.963668:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761715.963673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761715.963679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761715.963692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507536768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761715.963721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761715.963725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761715.963729:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761715/real 1789761715] req@ffff8d25d1b08700 x1876699507536256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761715.963729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761715.963740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1693 00000100:00080000:3.0:1789761715.963748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761715.963753:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761715.963754:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507536256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761715.963759:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761715.963766:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761715.963771:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761715.963775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507536896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761715.963780:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507536256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761715.963794:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507536384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761715.963816:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761715.963825:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761715/real 1789761715] req@ffff8d25d1b0b800 x1876699507536384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761715.963834:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507536384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761715.963840:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761715.963842:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761715.963845:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507536384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761715.963863:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507536512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761715.963874:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761715.963881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761715/real 1789761715] req@ffff8d25d1b08e00 x1876699507536512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761715.963889:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507536512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761715.963895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761715.963897:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761715.963899:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507536512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761715.963903:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507536640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761715.963912:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761715.963919:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761715/real 1789761715] req@ffff8d25d1b0a680 x1876699507536640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761715.963927:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507536640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761715.963932:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761715.963934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761715.963935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507536640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761715.963939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761715.963949:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761715.963974:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761715/real 1789761715] req@ffff8d25d1b09880 x1876699507536768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761715.963983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507536768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761715.963988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761715.963992:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761715.963995:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761715.963998:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761715.964011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761715.964018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761715/real 1789761715] req@ffff8d25d1b0bb80 x1876699507536896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761715.964026:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507536896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761740 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761715.964032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761715.964034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761715.964037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507536896:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761716.027557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761717.051634:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761718.075566:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761719.099592:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761720.123529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761721.083487:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761721.083496:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761721.083499:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761721.083507:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1698 00000100:00080000:0.0:1789761721.083515:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761721.083521:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761721.083549:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507537024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761721.083570:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761721.083573:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761721.083575:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761721.083578:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1698 00000100:00080000:0.0:1789761721.083581:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761721.083584:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761721.083592:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761721.083601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507537024:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761721.083603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507537152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761721.083615:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761721.083617:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761721.083619:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761721.083623:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1698 00000100:00080000:0.0:1789761721.083625:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761721.083628:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761721.083632:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761721.083640:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507537280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761721.083652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761721.083655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761721.083657:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761721.083673:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1698 00000800:00000100:3.0:1789761721.083677:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761721.083679:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761721.083684:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761721.083689:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761721.083700:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507537408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761721.083709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761721.083711:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761721.083713:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761721.083716:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1698 00000100:00080000:0.0:1789761721.083719:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761721.083721:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761721.083725:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761721.083726:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffff8d25d25ba300 x1876699507537024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761721.083737:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507537536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761721.083745:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761721.083746:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507537024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761721.083752:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761721.083757:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761721.083760:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761721.083766:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1698 00000100:00080000:3.0:1789761721.083767:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761721.083774:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507537024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761721.083775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761721.083784:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761721.083790:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761721.083796:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761721.083806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507537664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761721.083820:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761721.083834:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffff8d25d25b8000 x1876699507537152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.083844:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507537152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.083850:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.083853:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761721.083855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761721.083860:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761721.083872:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761721.083879:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffff8d25d25bbb80 x1876699507537280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.083887:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507537280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.083892:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.083894:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761721.083897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761721.083900:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761721.083911:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761721.083917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffff8d25d25b8a80 x1876699507537408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.083924:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507537408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.083929:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.083931:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761721.083933:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761721.083936:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507537536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761721.083946:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761721.083953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffff8d25d25b9180 x1876699507537536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.083960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507537536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.083966:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.083967:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761721.083969:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507537536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761721.083981:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507537664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761721.083991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761721.083997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761721/real 1789761721] req@ffff8d25d25ba680 x1876699507537664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761721.084005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507537664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761746 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761721.084010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761721.084011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761721.084013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507537664:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761721.147524:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761722.171521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761723.195499:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761724.219499:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761725.243509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761726.203480:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761726.203488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761726.203492:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761726.203500:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1703 00000100:00080000:2.0:1789761726.203508:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761726.203514:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761726.203543:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507537792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761726.203572:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761726.203577:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761726.203579:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761726.203584:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1703 00000100:00080000:2.0:1789761726.203588:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761726.203593:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761726.203604:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761726.203617:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507537792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761726.203637:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507537920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761726.203651:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761726.203655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761726.203657:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761726.203661:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1703 00000100:00080000:2.0:1789761726.203673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761726.203678:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761726.203683:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761726.203684:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761726.203697:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507538048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761726.203709:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761726.203712:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761726.203714:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761726.203717:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1703 00000100:00080000:2.0:1789761726.203721:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761726.203724:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffff8d25c8023800 x1876699507537792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761726.203727:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761726.203732:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761726.203744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507538176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761726.203780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761726.203783:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761726.203784:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761726.203789:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507537792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761726.203791:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1703 00000100:00080000:2.0:1789761726.203796:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761726.203802:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761726.203808:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761726.203809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.203814:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761726.203818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507538304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761726.203821:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507537792:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761726.203835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761726.203838:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761726.203839:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761726.203842:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1703 00000100:00080000:2.0:1789761726.203846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761726.203849:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537920:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761726.203851:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761726.203857:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761726.203866:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507538432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761726.203877:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761726.203890:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffff8d25c8022a00 x1876699507537920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.203899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507537920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.203905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.203908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761726.203910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507537920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761726.203914:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761726.203925:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761726.203932:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffff8d25c8022680 x1876699507538048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.203939:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507538048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.203949:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.203951:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761726.203953:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761726.203957:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761726.203966:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761726.203972:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffff8d25c8021180 x1876699507538176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.203979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507538176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.203985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.203986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761726.203988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761726.203992:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507538304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761726.204002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761726.204008:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffff8d25c8020a80 x1876699507538304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.204019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507538304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.204027:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.204030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761726.204034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507538304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761726.204051:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507538432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761726.204073:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761726.204082:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761726/real 1789761726] req@ffff8d25c8023800 x1876699507538432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761726.204090:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507538432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761751 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761726.204095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761726.204097:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761726.204100:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507538432:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761726.267506:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761727.291584:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761728.315530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761729.339584:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761730.363463:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761731.323457:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761731.323465:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761731.323469:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761731.323478:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1709 00000100:00080000:2.0:1789761731.323487:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761731.323493:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761731.323522:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507538560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761731.323550:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761731.323555:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761731.323557:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761731.323563:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1709 00000100:00080000:2.0:1789761731.323569:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761731.323575:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761731.323579:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507538560:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761731.323598:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761731.323614:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507538688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761731.323624:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761731.323627:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761731.323629:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761731.323634:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1709 00000100:00080000:2.0:1789761731.323638:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761731.323641:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761731.323645:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761731.323655:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507538816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761731.323662:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761731.323665:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761731.323667:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761731.323670:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1709 00000100:00080000:2.0:1789761731.323673:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761731.323676:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761731.323679:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761731.323692:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761731.323692:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507538944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761731.323711:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761731.323715:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761731.323717:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761731.323723:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1709 00000100:00080000:2.0:1789761731.323728:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761731.323734:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761731.323742:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538816:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789761731.323742:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761731.323761:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507539072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761731.323769:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761731.323772:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761731.323774:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761731.323778:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538944:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1789761731.323779:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1709 00000100:00080000:2.0:1789761731.323786:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761731.323790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761731.323795:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761731.323805:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761731.323808:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507539200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761731.323831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539200:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761731.387550:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761732.411483:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761733.435508:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789761734.395710:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1789761734.395759:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789761734.395780:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789761734.395837:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789761734.395850:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789761734.395862:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789761734.395874:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789761734.395888:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761734.395891:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761734] req@ffff8d25c8020000 x1876699507538944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761734.395926:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507538944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761734.395941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761734.395948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761734.395952:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761734.395961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761734] req@ffff8d25c8020700 x1876699507539072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761734.395974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507539072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761734.395981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761734.395985:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761734.395989:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761734.395995:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761734] req@ffff8d25c8023100 x1876699507538688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761734.396008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507538688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761734.396016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761734.396020:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761734.396024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761734.396031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761734] req@ffff8d25c8023b80 x1876699507538816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761734.396055:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507538816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761734.396063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761734.396067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761734.396070:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507538816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761734.396077:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761734] req@ffff8d25c8020380 x1876699507538560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761734.396090:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507538560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761734.396098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761734.396101:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761734.396105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507538560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761734.396141:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761731/real 1789761734] req@ffff8d25c8022d80 x1876699507539200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761734.396154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507539200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761756 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761734.396163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761734.396166:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761734.396170:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539200:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761734.459538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761735.483493:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761736.443558:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761736.443568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761736.443572:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761736.443583:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1714 00000100:00080000:3.0:1789761736.443592:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761736.443600:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761736.443633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507539328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761736.443660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761736.443664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761736.443666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761736.443671:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1714 00000100:00080000:3.0:1789761736.443674:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761736.443679:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761736.443690:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761736.443706:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507539328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761736.443707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507539456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761736.443730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761736.443734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761736.443737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761736.443741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1714 00000100:00080000:3.0:1789761736.443746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761736.443750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761736.443756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761736.443769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507539584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761736.443774:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761736.443785:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761736.443789:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761736.443791:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761736.443796:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1714 00000100:00080000:3.0:1789761736.443799:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761736.443803:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761736.443808:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761736.443820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507539712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761736.443824:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761736/real 1789761736] req@ffff8d25d1b08a80 x1876699507539328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761736.443839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761736.443845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761736.443847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761736.443852:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1714 00000100:00100000:0.0:1789761736.443856:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507539328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761736.443857:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761736.443865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761736.443874:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761736.443876:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761736.443883:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761736.443887:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507539840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761736.443892:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507539328:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761736.443906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761736.443909:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761736.443911:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761736.443916:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1714 00000100:00080000:3.0:1789761736.443919:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761736.443923:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761736.443928:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761736.443933:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507539456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761736.443940:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507539968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761736.443967:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761736.443983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761736/real 1789761736] req@ffff8d25d1b0aa00 x1876699507539456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761736.443996:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507539456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761736.444006:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761736.444009:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761736.444013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507539456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761736.444019:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507539584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761736.444039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761736.444051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761736/real 1789761736] req@ffff8d25d1b09180 x1876699507539584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761736.444064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507539584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761736.444080:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761736.444083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761736.444087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507539584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761736.444094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507539712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761736.444116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761736.444128:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761736/real 1789761736] req@ffff8d25d1b0ad80 x1876699507539712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761736.444139:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507539712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761736.444147:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761736.444150:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761736.444153:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507539712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761736.444159:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761736.444180:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761736.444190:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761736/real 1789761736] req@ffff8d25d1b09c00 x1876699507539840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761736.444200:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507539840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761736.444207:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761736.444210:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761736.444213:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761736.444246:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761736.444267:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761736.444279:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761736/real 1789761736] req@ffff8d25d1b0b100 x1876699507539968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761736.444291:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507539968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761761 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761736.444299:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761736.444302:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761736.444305:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507539968:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761736.507513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761737.531588:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761738.555550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761739.579542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761740.603588:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761741.563446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761741.563454:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761741.563458:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761741.563467:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1719 00000100:00080000:3.0:1789761741.563475:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761741.563481:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761741.563535:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507540096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761741.563569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761741.563573:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761741.563575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761741.563580:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1719 00000100:00080000:3.0:1789761741.563584:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761741.563589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761741.563599:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761741.563614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507540224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761741.563618:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507540096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761741.563632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761741.563634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761741.563636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761741.563639:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1719 00000100:00080000:3.0:1789761741.563642:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761741.563645:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761741.563649:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761741.563659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507540352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761741.563666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761741.563669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761741.563670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761741.563672:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1719 00000100:00080000:3.0:1789761741.563674:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761741.563679:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761741.563679:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761741.563686:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761741.563694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507540480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761741.563702:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761741.563705:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761741.563708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761741.563714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1719 00000100:00100000:0.0:1789761741.563719:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761741/real 1789761741] req@ffff8d25d1b0b480 x1876699507540096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761741.563720:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761741.563730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761741.563739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761741.563742:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507540096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761741.563753:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789761741.563758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507540608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761741.563759:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761741.563767:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507540096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761741.563783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761741.563787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761741.563789:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761741.563794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1719 00000100:00080000:3.0:1789761741.563798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761741.563801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761741.563806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761741.563811:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761741.563818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507540736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761741.563839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761741.563852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761741/real 1789761741] req@ffff8d25d1b09500 x1876699507540224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761741.563861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507540224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761741.563868:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761741.563870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761741.563873:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761741.563876:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761741.563888:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761741.563895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761741/real 1789761741] req@ffff8d25d1b08000 x1876699507540352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761741.563903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507540352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761741.563908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761741.563910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761741.563912:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761741.563915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761741.563926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761741.563932:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761741/real 1789761741] req@ffff8d25d1b08380 x1876699507540480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761741.563940:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507540480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761741.563946:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761741.563955:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761741.563958:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761741.563962:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507540608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761741.563974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761741.563981:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761741/real 1789761741] req@ffff8d25d1b0a300 x1876699507540608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761741.563988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507540608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761741.563994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761741.563995:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761741.563997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507540608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761741.564014:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507540736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761741.564025:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761741.564032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761741/real 1789761741] req@ffff8d25d1b0b480 x1876699507540736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761741.564041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507540736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761766 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761741.564050:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761741.564053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761741.564058:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507540736:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761741.627494:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761742.651530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761743.675506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761744.699504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761745.723495:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761746.683449:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761746.683459:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761746.683464:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761746.683485:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1724 00000100:00080000:3.0:1789761746.683495:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761746.683502:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761746.683538:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507540864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761746.683565:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761746.683569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761746.683570:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761746.683575:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1724 00000100:00080000:3.0:1789761746.683580:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761746.683585:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761746.683594:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761746.683610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507540992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761746.683621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761746.683625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761746.683627:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761746.683632:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1724 00000100:00080000:3.0:1789761746.683636:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761746.683640:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761746.683645:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761746.683652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507540864:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761746.683669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507541120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761746.683688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761746.683692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761746.683694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761746.683699:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1724 00000100:00080000:3.0:1789761746.683704:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761746.683707:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761746.683712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761746.683724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507541248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761746.683735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761746.683739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761746.683740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761746.683752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1724 00000800:00000100:0.0:1789761746.683756:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761746.683759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761746.683765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761746.683770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761746.683783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507541376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761746.683793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761746.683797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761746.683799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761746.683804:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1724 00000100:00080000:3.0:1789761746.683807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761746.683812:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761746.683818:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761746.683824:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761746/real 1789761746] req@ffff8d25d1b0b100 x1876699507540864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761746.683834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507541504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761746.683866:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507540864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761746.683880:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761746.683886:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761746.683890:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507540864:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761746.683899:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761746.683926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761746.683940:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761746/real 1789761746] req@ffff8d25d1b09c00 x1876699507540992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761746.683953:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507540992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761746.683962:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761746.683965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761746.683969:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507540992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761746.684002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761746.684022:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761746.684036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761746/real 1789761746] req@ffff8d25d1b0ad80 x1876699507541120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761746.684047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507541120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761746.684055:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761746.684058:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761746.684062:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761746.684078:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761746.684099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761746.684110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761746/real 1789761746] req@ffff8d25d1b09180 x1876699507541248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761746.684122:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507541248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761746.684131:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761746.684134:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761746.684137:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761746.684143:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507541376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761746.684161:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761746.684174:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761746/real 1789761746] req@ffff8d25d1b0aa00 x1876699507541376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761746.684185:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507541376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761746.684195:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761746.684198:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761746.684201:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507541376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761746.684207:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507541504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761746.684226:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761746.684237:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761746/real 1789761746] req@ffff8d25d1b08a80 x1876699507541504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761746.684248:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507541504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761771 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761746.684256:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761746.684259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761746.684262:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507541504:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761746.747509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761747.771491:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761748.795520:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761749.819493:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761750.843539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761751.803488:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761751.803498:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761751.803503:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761751.803513:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1729 00000100:00080000:3.0:1789761751.803523:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761751.803531:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761751.803565:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507541632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761751.803593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761751.803597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761751.803599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761751.803603:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1729 00000100:00080000:3.0:1789761751.803607:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761751.803612:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761751.803621:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761751.803635:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507541632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761751.803638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507541760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761751.803656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761751.803658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761751.803660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761751.803664:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1729 00000100:00080000:3.0:1789761751.803666:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761751.803670:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761751.803674:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761751.803683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507541888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761751.803690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761751.803693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761751.803694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761751.803697:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1729 00000100:00080000:3.0:1789761751.803700:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761751.803704:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789761751.803710:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789761751.803713:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761751.803729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507542016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761751.803741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761751.803744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761751.803747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761751.803752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1729 00000100:00100000:0.0:1789761751.803755:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761751/real 1789761751] req@ffff8d25d1b0bb80 x1876699507541632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761751.803758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761751.803766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761751.803776:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761751.803785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507541632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761751.803794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507542144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761751.803796:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761751.803804:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761751.803812:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507541632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761751.803818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761751.803823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761751.803826:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761751.803831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1729 00000100:00100000:0.0:1789761751.803836:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541760:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761751.803837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761751.803846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761751.803854:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761751.803864:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761751.803871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507542272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761751.803885:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761751/real 1789761751] req@ffff8d25d1b09880 x1876699507541760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761751.803899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507541760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761751.803906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761751.803909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761751.803912:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761751.803918:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761751.803937:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761751.803949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761751/real 1789761751] req@ffff8d25d1b0a680 x1876699507541888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761751.803963:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507541888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761751.803972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761751.803976:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761751.803980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507541888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761751.803987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761751.804006:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761751.804016:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761751/real 1789761751] req@ffff8d25d1b08e00 x1876699507542016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761751.804025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507542016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761751.804031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761751.804033:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761751.804036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761751.804039:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507542144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761751.804050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761751.804056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761751/real 1789761751] req@ffff8d25d1b0b800 x1876699507542144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761751.804064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507542144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761751.804073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761751.804076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761751.804088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507542144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761751.804113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507542272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761751.804136:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761751.804148:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761751/real 1789761751] req@ffff8d25d1b08700 x1876699507542272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761751.804162:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507542272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761776 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761751.804170:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761751.804172:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761751.804175:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507542272:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761751.867552:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761752.891572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761753.915526:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761754.939513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761755.963532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761756.923461:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761756.923469:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761756.923473:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761756.923481:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1734 00000100:00080000:3.0:1789761756.923489:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761756.923495:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761756.923526:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507542400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761756.923546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761756.923549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761756.923550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761756.923554:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1734 00000100:00080000:3.0:1789761756.923557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761756.923559:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761756.923567:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761756.923576:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507542528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761756.923583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761756.923585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761756.923587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761756.923590:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1734 00000100:00080000:3.0:1789761756.923592:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761756.923597:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507542400:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761756.923598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761756.923607:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761756.923630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507542656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761756.923641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761756.923645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761756.923648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761756.923653:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1734 00000800:00000100:0.0:1789761756.923658:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761756.923660:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761756.923667:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761756.923673:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761756.923690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507542784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761756.923696:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761756/real 1789761756] req@ffff8d25d1b08700 x1876699507542400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761756.923710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761756.923715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761756.923717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761756.923719:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507542400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761756.923724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1734 00000100:00080000:3.0:1789761756.923730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761756.923735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761756.923736:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789761756.923741:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761756.923742:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761756.923749:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507542400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761756.923758:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507542912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761756.923759:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761756.923774:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761756.923777:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761756.923779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761756.923782:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1734 00000100:00080000:3.0:1789761756.923786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761756.923787:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761756.923791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761756.923796:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761756.923809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507543040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761756.923810:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761756/real 1789761756] req@ffff8d25d1b0b800 x1876699507542528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761756.923843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507542528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761756.923850:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761756.923853:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761756.923856:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761756.923880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761756.923903:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761756.923916:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761756/real 1789761756] req@ffff8d25d1b08e00 x1876699507542656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761756.923930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507542656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761756.923937:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761756.923939:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761756.923942:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761756.923947:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761756.923962:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761756.923972:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761756/real 1789761756] req@ffff8d25d1b0a680 x1876699507542784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761756.923985:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507542784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761756.923994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761756.923998:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761756.924002:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507542784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761756.924009:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507542912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761756.924028:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761756.924037:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761756/real 1789761756] req@ffff8d25d1b09880 x1876699507542912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761756.924046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507542912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761756.924051:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761756.924054:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761756.924056:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507542912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761756.924059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507543040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761756.924071:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761756.924077:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761756/real 1789761756] req@ffff8d25d1b0bb80 x1876699507543040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761756.924084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507543040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761781 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761756.924090:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761756.924091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761756.924093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507543040:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761756.987536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761758.011564:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761759.035520:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761760.059559:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761761.083497:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761762.043495:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.043505:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761762.043509:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.043519:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1739 00000100:00080000:2.0:1789761762.043530:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.043538:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761762.043571:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507543168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761762.043598:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.043601:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761762.043603:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.043608:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1739 00000100:00080000:2.0:1789761762.043611:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.043615:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761762.043624:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761762.043637:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507543296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761762.043642:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507543168:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761762.043657:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.043660:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761762.043661:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.043666:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1739 00000100:00080000:2.0:1789761762.043668:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.043672:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761762.043677:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761762.043687:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507543424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761762.043694:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.043696:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761762.043697:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.043703:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1739 00000800:00000100:0.0:1789761762.043704:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761762.043707:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.043711:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761762.043714:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761762.043723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507543552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761762.043738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761762.043741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761762.043742:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.043747:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1739 00000100:00100000:0.0:1789761762.043748:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffff8d25c8022300 x1876699507543168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761762.043750:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.043756:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761762.043761:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761762.043774:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507543680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761762.043782:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761762.043783:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507543168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761762.043788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761762.043792:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761762.043809:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1739 00000100:00080000:2.0:1789761762.043813:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761762.043821:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761762.043823:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789761762.043829:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761762.043832:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761762.043839:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507543168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761762.043847:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507543808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761762.043868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507543296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761762.043892:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761762.043903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffff8d25c8023480 x1876699507543296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761762.043912:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507543296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761762.043918:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761762.043921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761762.043923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507543296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761762.043927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507543424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761762.043938:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761762.043945:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffff8d25c8020e00 x1876699507543424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761762.043952:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507543424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761762.043958:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761762.043959:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761762.043961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507543424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761762.043966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507543552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761762.043977:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761762.043983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffff8d25c8021500 x1876699507543552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761762.043991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507543552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761762.043997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761762.043998:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761762.044000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507543552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761762.044003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507543680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761762.044013:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761762.044020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffff8d25c8021c00 x1876699507543680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761762.044027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507543680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761762.044032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761762.044034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761762.044036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507543680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761762.044039:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507543808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761762.044048:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761762.044054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761762/real 1789761762] req@ffff8d25c8021880 x1876699507543808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761762.044062:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507543808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761787 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761762.044067:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761762.044069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761762.044071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507543808:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761762.107539:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761763.131495:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761764.155534:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761765.179511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761766.203478:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761767.163525:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.163535:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761767.163539:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761767.163551:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1744 00000100:00080000:0.0:1789761767.163560:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.163570:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761767.163608:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507543936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761767.163645:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.163651:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761767.163654:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761767.163660:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1744 00000100:00080000:0.0:1789761767.163665:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.163671:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761767.163682:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761767.163699:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507544064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761767.163719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.163723:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761767.163725:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761767.163730:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1744 00000100:00080000:0.0:1789761767.163735:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.163740:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761767.163756:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761767.163772:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507544192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761767.163784:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.163788:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761767.163791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761767.163795:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1744 00000100:00080000:0.0:1789761767.163799:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.163803:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761767.163810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761767.163823:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507544320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761767.163835:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.163839:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761767.163842:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761767.163844:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507543936:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761767.163848:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1744 00000100:00080000:0.0:1789761767.163855:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.163860:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761767.163866:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761767.163882:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507544448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761767.163893:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761767.163897:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761767.163899:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761767.163902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761767.163907:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1744 00000100:00080000:0.0:1789761767.163913:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761767.163918:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761767.163924:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761767.163937:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507544576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761767.164047:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffff8d25d25bb800 x1876699507543936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.164101:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507543936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.164110:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.164113:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761767.164117:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507543936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761767.164123:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761767.164143:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761767.164153:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffff8d25d25b9f80 x1876699507544064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.164161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507544064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.164167:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.164169:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761767.164172:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761767.164190:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761767.164202:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761767.164209:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffff8d25d25bad80 x1876699507544192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.164221:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507544192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.164230:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.164234:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761767.164238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761767.164244:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761767.164265:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761767.164277:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffff8d25d25b8380 x1876699507544320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.164287:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507544320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.164293:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.164295:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761767.164297:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761767.164301:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507544448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761767.164326:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761767.164335:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffff8d25d25b8700 x1876699507544448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.164343:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507544448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.164349:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.164352:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761767.164354:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507544448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761767.164357:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507544576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761767.164369:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761767.164404:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761767/real 1789761767] req@ffff8d25d25bb100 x1876699507544576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761767.164417:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507544576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761792 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761767.164423:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761767.164425:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761767.164428:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507544576:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761767.227492:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761768.251531:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761769.275479:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761770.299581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761771.323514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761772.283458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.283468:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761772.283472:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.283482:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1749 00000100:00080000:0.0:1789761772.283491:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.283503:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761772.283536:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507544704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761772.283563:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.283567:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761772.283569:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.283574:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1749 00000100:00080000:0.0:1789761772.283578:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.283583:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761772.283592:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761772.283607:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507544832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761772.283612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507544704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761772.283628:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.283632:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761772.283634:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.283639:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1749 00000100:00080000:0.0:1789761772.283643:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.283648:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761772.283653:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761772.283670:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507544960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761772.283682:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789761772.283682:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761772.283689:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761772.283691:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.283697:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1749 00000100:00080000:0.0:1789761772.283705:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.283709:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761772.283715:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761772.283723:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffff8d25d25bb480 x1876699507544704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761772.283731:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507545088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761772.283743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507544704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761772.283746:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.283756:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761772.283760:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761772.283768:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1749 00000100:00080000:0.0:1789761772.283775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761772.283777:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761772.283779:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761772.283786:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789761772.283787:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761772.283794:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507544704:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761772.283802:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507545216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761772.283813:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761772.283817:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761772.283819:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761772.283823:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544832:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761772.283827:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1749 00000100:00080000:0.0:1789761772.283833:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761772.283836:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761772.283842:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761772.283855:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507545344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761772.283864:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761772.283887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffff8d25d25b8e00 x1876699507544832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.283903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507544832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.283912:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.283916:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761772.283920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761772.283927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761772.283951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761772.283965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffff8d25d25b9c00 x1876699507544960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.283979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507544960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.283988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.283992:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761772.283996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507544960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761772.284004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761772.284027:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761772.284041:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffff8d25d25bb100 x1876699507545088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.284054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507545088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.284063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.284067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761772.284070:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761772.284077:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507545216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761772.284100:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761772.284112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffff8d25d25b8700 x1876699507545216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.284125:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507545216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.284134:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.284137:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761772.284141:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507545216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761772.284161:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507545344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761772.284185:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761772.284197:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761772/real 1789761772] req@ffff8d25d25b8380 x1876699507545344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761772.284211:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507545344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761797 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761772.284220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761772.284223:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761772.284227:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507545344:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761772.347552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761773.371514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761774.395647:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761775.419550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761776.443562:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761777.403493:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761777.403501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761777.403504:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761777.403513:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1755 00000100:00080000:3.0:1789761777.403522:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761777.403530:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761777.403566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507545472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761777.403593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761777.403599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761777.403600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761777.403605:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1755 00000100:00080000:3.0:1789761777.403609:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761777.403613:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761777.403623:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761777.403625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507545472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761777.403640:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507545600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761777.403654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761777.403658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761777.403660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761777.403664:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1755 00000100:00080000:3.0:1789761777.403669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761777.403673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761777.403679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761777.403690:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761777.403695:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507545728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761777.403704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761777.403707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761777.403709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761777.403713:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1755 00000100:00080000:3.0:1789761777.403717:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761777.403722:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761777.403727:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761777/real 1789761777] req@ffff8d25d1b08a80 x1876699507545472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789761777.403730:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761777.403754:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507545472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761777.403754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507545856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761777.403785:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761777.403789:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761777.403789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761777.403796:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507545472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761777.403798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761777.403803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761777.403809:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1755 00000100:00080000:3.0:1789761777.403814:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761777.403819:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545600:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761777.403820:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761777.403830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789761777.403844:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761777.403848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507545984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761777.403858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761777.403862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761777.403863:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761777/real 1789761777] req@ffff8d25d1b0aa00 x1876699507545600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761777.403864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761777.403870:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1755 00000100:00080000:3.0:1789761777.403875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761777.403883:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761777.403891:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761777.403896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507545600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761777.403908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507546112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761777.403911:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761777.403921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761777.403930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761777.403939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761777.403958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761777.403967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761777/real 1789761777] req@ffff8d25d1b09180 x1876699507545728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761777.403980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507545728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761777.403986:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761777.403988:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761777.403991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761777.404002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761777.404015:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761777.404023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761777/real 1789761777] req@ffff8d25d1b0ad80 x1876699507545856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761777.404031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507545856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761777.404037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761777.404039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761777.404041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507545856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761777.404055:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507545984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761777.404065:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761777.404072:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761777/real 1789761777] req@ffff8d25d1b09c00 x1876699507545984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761777.404079:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507545984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761777.404089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761777.404091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761777.404094:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507545984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761777.404097:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507546112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761777.404106:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761777.404112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761777/real 1789761777] req@ffff8d25d1b0b100 x1876699507546112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761777.404119:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507546112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761802 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761777.404125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761777.404126:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761777.404128:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507546112:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761777.467545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761778.491535:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761779.515592:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761780.539545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761781.563568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761782.523475:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761782.523485:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761782.523490:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761782.523499:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1760 00000100:00080000:2.0:1789761782.523509:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761782.523516:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761782.523554:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507546240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761782.523582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761782.523586:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761782.523588:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761782.523593:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1760 00000100:00080000:2.0:1789761782.523598:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761782.523603:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761782.523613:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761782.523623:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507546240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761782.523632:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507546368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761782.523646:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761782.523650:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761782.523652:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761782.523656:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1760 00000100:00080000:2.0:1789761782.523660:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761782.523664:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761782.523671:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761782.523683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507546496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761782.523693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789761782.523697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761782.523699:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761782.523704:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761782.523710:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1760 00000100:00080000:2.0:1789761782.523715:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761782.523719:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761782.523726:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761782.523738:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761782/real 1789761782] req@ffff8d25c8021880 x1876699507546240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761782.523742:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507546624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761782.523765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761782.523769:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507546240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761782.523771:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761782.523775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761782.523786:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1760 00000100:00080000:0.0:1789761782.523793:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761782.523794:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761782.523804:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789761782.523809:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789761782.523810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761782.523818:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507546240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761782.523828:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507546752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761782.523840:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761782.523844:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789761782.523848:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507546368:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761782.523848:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761782.523855:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1760 00000100:00080000:2.0:1789761782.523859:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761782.523865:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761782.523872:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761782.523885:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507546880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761782.523897:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761782.523916:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761782/real 1789761782] req@ffff8d25c8021c00 x1876699507546368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761782.523932:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507546368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761782.523942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761782.523946:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761782.523950:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507546368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761782.523964:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507546496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761782.523984:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761782.523995:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761782/real 1789761782] req@ffff8d25c8021500 x1876699507546496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761782.524008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507546496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761782.524017:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761782.524021:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761782.524024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507546496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761782.524030:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507546624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761782.524045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761782.524055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761782/real 1789761782] req@ffff8d25c8020e00 x1876699507546624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761782.524068:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507546624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761782.524077:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761782.524081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761782.524084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507546624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761782.524090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507546752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761782.524104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761782.524114:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761782/real 1789761782] req@ffff8d25c8023480 x1876699507546752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761782.524131:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507546752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761782.524137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761782.524140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761782.524144:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507546752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761782.524163:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507546880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761782.524184:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761782.524197:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761782/real 1789761782] req@ffff8d25c8022300 x1876699507546880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761782.524222:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507546880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761807 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761782.524231:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761782.524235:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761782.524239:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507546880:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761782.587536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761783.611512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761784.636462:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761785.659547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761786.683475:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761787.643471:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761787.643480:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761787.643484:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761787.643493:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1765 00000100:00080000:2.0:1789761787.643501:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761787.643507:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761787.643536:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507547008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761787.643556:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761787.643559:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761787.643560:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761787.643563:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1765 00000100:00080000:2.0:1789761787.643565:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761787.643568:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761787.643575:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761787.643583:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507547136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761787.643590:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761787.643592:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761787.643593:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761787.643596:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1765 00000100:00080000:2.0:1789761787.643600:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761787.643602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507547008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761787.643602:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761787.643608:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761787.643616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507547264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761787.643625:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761787.643628:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761787.643630:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761787.643633:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1765 00000100:00080000:2.0:1789761787.643637:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761787.643641:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761787.643646:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761787.643656:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507547392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761787.643664:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761787.643673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761787.643676:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761787.643678:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761787.643683:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1765 00000100:00080000:2.0:1789761787.643686:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761787.643690:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761787.643695:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761787.643702:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761787/real 1789761787] req@ffff8d25c8022d80 x1876699507547008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761787.643709:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507547520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761787.643718:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507547008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761787.643720:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761787.643728:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761787.643732:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761787.643733:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761787.643738:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1765 00000100:00080000:0.0:1789761787.643739:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761787.643746:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761787.643747:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507547008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761787.643754:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761787.643761:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761787.643773:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507547648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761787.643778:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761787.643810:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761787.643825:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761787/real 1789761787] req@ffff8d25c8020380 x1876699507547136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761787.643833:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507547136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761787.643838:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761787.643840:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761787.643843:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761787.643847:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761787.643860:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761787.643867:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761787/real 1789761787] req@ffff8d25c8023b80 x1876699507547264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761787.643874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507547264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761787.643879:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761787.643881:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761787.643883:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761787.643886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761787.643900:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761787.643909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761787/real 1789761787] req@ffff8d25c8023100 x1876699507547392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761787.643921:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507547392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761787.643929:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761787.643932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761787.643936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761787.643941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507547520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761787.643955:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761787.643963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761787/real 1789761787] req@ffff8d25c8020700 x1876699507547520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761787.643974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507547520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761787.643981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761787.643984:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761787.643987:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507547520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761787.644001:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507547648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761787.644013:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761787.644020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761787/real 1789761787] req@ffff8d25c8020000 x1876699507547648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761787.644027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507547648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761812 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761787.644032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761787.644034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761787.644035:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507547648:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761787.707501:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761788.731510:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761789.755523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761790.779503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761791.803504:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761792.763471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761792.763479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761792.763483:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761792.763500:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1770 00000100:00080000:2.0:1789761792.763509:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761792.763520:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761792.763554:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507547776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761792.763591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761792.763595:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761792.763597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761792.763601:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1770 00000100:00080000:2.0:1789761792.763606:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761792.763611:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761792.763620:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761792.763630:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507547776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761792.763637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507547904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761792.763650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761792.763654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761792.763656:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761792.763660:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1770 00000100:00080000:2.0:1789761792.763663:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761792.763667:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761792.763672:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761792.763681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507548032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761792.763688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761792.763691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761792.763693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761792.763697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761792.763698:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1770 00000100:00080000:2.0:1789761792.763703:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761792.763706:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761792.763711:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761792.763723:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507548160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761792.763734:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761792.763738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761792.763740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761792.763743:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761792/real 1789761792] req@ffff8d25c8023800 x1876699507547776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761792.763749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1770 00000100:00080000:2.0:1789761792.763758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761792.763763:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507547776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761792.763764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761792.763772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761792.763782:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761792.763787:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761792.763794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507548288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761792.763795:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507547776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761792.763808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761792.763811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761792.763813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761792.763816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1770 00000100:00080000:2.0:1789761792.763820:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761792.763822:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761792.763826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761792.763831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761792.763841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507548416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761792.763853:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761792.763871:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761792/real 1789761792] req@ffff8d25c8020a80 x1876699507547904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761792.763886:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507547904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761792.763896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761792.763900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761792.763903:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507547904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761792.763908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761792.763925:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761792.763933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761792/real 1789761792] req@ffff8d25c8021180 x1876699507548032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761792.763941:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507548032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761792.763947:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761792.763950:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761792.763952:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761792.763956:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761792.763968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761792.763975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761792/real 1789761792] req@ffff8d25c8022680 x1876699507548160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761792.763982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507548160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761792.763988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761792.763990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761792.763992:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761792.763995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507548288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761792.764005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761792.764011:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761792/real 1789761792] req@ffff8d25c8022a00 x1876699507548288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761792.764019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507548288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761792.764024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761792.764026:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761792.764028:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507548288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761792.764039:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507548416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761792.764050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761792.764056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761792/real 1789761792] req@ffff8d25c8023800 x1876699507548416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761792.764064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507548416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761817 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761792.764070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761792.764071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761792.764073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507548416:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761792.827522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761793.851532:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761794.875531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761795.899506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761796.923527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761797.883473:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761797.883484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761797.883489:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761797.883499:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1775 00000100:00080000:1.0:1789761797.883507:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761797.883515:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761797.883552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507548544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761797.883573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761797.883577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761797.883579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761797.883584:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1775 00000100:00080000:1.0:1789761797.883588:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761797.883593:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761797.883602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507548544:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789761797.883602:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761797.883616:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507548672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761797.883626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761797.883630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761797.883631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761797.883635:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1775 00000100:00080000:1.0:1789761797.883651:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761797.883655:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761797.883660:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761797.883673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507548800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761797.883682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761797.883685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761797.883687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761797.883692:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1775 00000100:00080000:1.0:1789761797.883695:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761797.883696:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761797.883700:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761797.883704:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761797.883717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507548928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761797.883728:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761797.883729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761797.883733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761797.883735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761797.883739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1775 00000100:00080000:1.0:1789761797.883746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761797.883750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761797.883756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761797.883761:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789761797.883766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507549056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761797.883776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761797.883779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761797.883782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761797.883787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1775 00000100:00100000:0.0:1789761797.883788:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761797.883790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761797.883794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761797.883799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761797.883810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507549184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761797.883843:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549184:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761797.947504:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761798.971532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761799.995577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761800.955622:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1789761800.955673:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761800.955690:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761800.955731:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761800.955742:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761800.955752:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761800.955765:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761800.955777:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789761800.955787:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761797/real 1789761800] req@ffff8d25c816aa00 x1876699507549056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761800.955815:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507549056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761800.955828:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761800.955833:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761800.955838:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761800.955849:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761797/real 1789761800] req@ffff8d25c816a680 x1876699507548928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761800.955863:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507548928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761800.955871:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761800.955876:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761800.955879:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761800.955887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761797/real 1789761800] req@ffff8d25c8168e00 x1876699507548672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761800.955900:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507548672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761800.955908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761800.955912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761800.955915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761800.955922:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761797/real 1789761800] req@ffff8d25c8169f80 x1876699507548800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761800.955934:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507548800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761800.955942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761800.955945:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761800.955948:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507548800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761800.955955:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761797/real 1789761800] req@ffff8d25c8168700 x1876699507548544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761800.955967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507548544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761800.955975:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761800.955978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761800.955982:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507548544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761800.956012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761797/real 1789761800] req@ffff8d25c8169500 x1876699507549184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761800.956025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507549184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761822 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761800.956033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761800.956037:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761800.956040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549184:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761801.019561:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761802.043501:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761803.003521:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761803.003529:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761803.003533:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761803.003541:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1780 00000100:00080000:3.0:1789761803.003551:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761803.003558:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761803.003609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507549312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761803.003657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761803.003662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761803.003664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761803.003669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1780 00000100:00080000:3.0:1789761803.003675:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761803.003680:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761803.003691:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761803.003699:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507549312:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761803.003710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507549440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761803.003725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761803.003729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761803.003731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761803.003736:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1780 00000100:00080000:3.0:1789761803.003755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761803.003760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761803.003766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761803.003772:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761803.003783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507549568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761803.003795:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761803.003798:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761803.003800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761803.003804:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1780 00000100:00080000:3.0:1789761803.003808:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761803.003812:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761803.003819:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761803.003821:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffff8d25d1b0b480 x1876699507549312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761803.003835:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507549696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761803.003847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507549312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761803.003850:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761803.003861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761803.003866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761803.003871:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761803.003874:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1780 00000100:00080000:2.0:1789761803.003880:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761803.003882:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761803.003887:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507549312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761803.003892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761803.003900:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761803.003913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507549824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761803.003917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507549440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789761803.003934:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761803.003937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761803.003940:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761803.003946:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1780 00000100:00080000:3.0:1789761803.003950:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761803.003954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761803.003959:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761803.003966:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761803.003975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffff8d25d1b0a300 x1876699507549440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761803.003982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507549952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761803.003999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507549440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761803.004010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761803.004014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761803.004019:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507549440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761803.004026:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507549568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761803.004061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761803.004074:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffff8d25d1b08380 x1876699507549568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761803.004086:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507549568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761803.004094:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761803.004098:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761803.004101:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507549568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761803.004107:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507549696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761803.004127:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761803.004138:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffff8d25d1b08000 x1876699507549696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761803.004149:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507549696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761803.004157:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761803.004161:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761803.004164:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507549696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761803.004186:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761803.004208:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761803.004219:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffff8d25d1b09500 x1876699507549824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761803.004232:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507549824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761803.004240:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761803.004244:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761803.004247:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761803.004254:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761803.004275:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761803.004289:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761803/real 1789761803] req@ffff8d25d1b0b480 x1876699507549952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761803.004300:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507549952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761828 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761803.004309:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761803.004311:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761803.004315:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507549952:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761803.067507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761804.091496:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761805.115537:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761806.139523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761807.163545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761808.123490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761808.123498:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761808.123502:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761808.123511:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1785 00000100:00080000:3.0:1789761808.123520:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761808.123527:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761808.123561:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507550080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761808.123579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761808.123583:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761808.123585:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761808.123588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1785 00000100:00080000:3.0:1789761808.123591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761808.123594:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761808.123602:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761808.123609:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507550080:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761808.123613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507550208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761808.123622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761808.123625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761808.123626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761808.123629:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1785 00000100:00080000:3.0:1789761808.123631:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761808.123633:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761808.123636:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761808.123645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507550336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761808.123651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761808.123653:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761808.123655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761808.123658:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1785 00000100:00080000:3.0:1789761808.123660:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761808.123662:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761808.123667:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789761808.123672:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761808.123677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507550464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761808.123685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761808.123688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761808.123689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761808.123691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1785 00000100:00080000:3.0:1789761808.123694:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761808.123696:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761808.123699:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761808.123706:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507550592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761808.123712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761808.123714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761808.123716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789761808.123720:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffff8d25d1b0b100 x1876699507550080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761808.123721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1785 00000100:00080000:3.0:1789761808.123726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761808.123733:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761808.123739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761808.123746:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507550080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761808.123753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507550720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761808.123757:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761808.123772:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761808.123777:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507550080:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761808.123799:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761808.123818:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761808.123828:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffff8d25d1b09c00 x1876699507550208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761808.123836:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507550208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761808.123846:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761808.123849:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761808.123851:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761808.123855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761808.123866:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761808.123872:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffff8d25d1b0ad80 x1876699507550336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761808.123879:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507550336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761808.123885:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761808.123887:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761808.123889:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761808.123892:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761808.123902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761808.123908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffff8d25d1b09180 x1876699507550464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761808.123916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507550464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761808.123922:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761808.123926:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761808.123930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761808.123936:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507550592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761808.123955:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761808.123968:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffff8d25d1b0aa00 x1876699507550592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761808.123980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507550592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761808.123985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761808.123987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761808.123989:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507550592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761808.123994:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507550720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761808.124028:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761808.124041:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761808/real 1789761808] req@ffff8d25d1b08a80 x1876699507550720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761808.124054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507550720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761833 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761808.124062:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761808.124065:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761808.124067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507550720:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761808.187496:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761809.211491:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761810.235474:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761811.259531:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761812.283493:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761813.243490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.243501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761813.243505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761813.243516:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1790 00000100:00080000:0.0:1789761813.243529:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.243536:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761813.243587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507550848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761813.243618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.243621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761813.243622:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761813.243626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1790 00000100:00080000:0.0:1789761813.243628:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.243632:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761813.243642:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761813.243660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507550976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761813.243666:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507550848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761813.243677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.243681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761813.243682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761813.243686:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1790 00000100:00080000:0.0:1789761813.243689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.243692:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761813.243697:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761813.243715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507551104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761813.243726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.243731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761813.243733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761813.243735:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761813.243748:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1790 00000100:00080000:0.0:1789761813.243756:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.243762:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761813.243768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761813.243788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507551232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761813.243790:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffff8d25d25bad80 x1876699507550848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761813.243840:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761813.243845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761813.243847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761813.243851:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1790 00000100:00080000:0.0:1789761813.243854:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.243860:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761813.243862:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507550848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761813.243865:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761813.243878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507551360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761813.243894:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761813.243895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761813.243901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761813.243905:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761813.243910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761813.243915:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1790 00000100:00080000:0.0:1789761813.243921:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761813.243928:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761813.243930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507550848:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761813.243933:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761813.243948:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507551488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761813.243986:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761813.244012:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761813.244023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffff8d25d25b9f80 x1876699507550976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.244032:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507550976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.244041:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.244045:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761813.244048:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507550976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761813.244055:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761813.244077:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761813.244090:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffff8d25d25bb800 x1876699507551104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.244105:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507551104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.244111:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.244114:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761813.244116:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761813.244120:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761813.244136:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761813.244143:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffff8d25d25ba680 x1876699507551232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.244150:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507551232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.244156:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.244158:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761813.244160:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761813.244164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507551360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761813.244174:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761813.244180:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffff8d25d25b9180 x1876699507551360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.244187:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507551360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.244196:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.244199:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761813.244201:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507551360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761813.244204:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507551488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761813.244214:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761813.244220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761813/real 1789761813] req@ffff8d25d25b8a80 x1876699507551488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761813.244227:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507551488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761838 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761813.244231:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761813.244233:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761813.244235:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507551488:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761813.307541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761814.331565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761815.355513:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761816.379549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761817.403536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761818.363526:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.363537:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761818.363542:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.363553:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1796 00000100:00080000:2.0:1789761818.363562:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.363574:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761818.363611:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507551616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761818.363648:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.363652:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761818.363655:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.363661:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1796 00000100:00080000:2.0:1789761818.363668:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.363672:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761818.363681:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761818.363694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507551616:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761818.363695:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507551744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761818.363719:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.363721:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761818.363723:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.363727:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1796 00000100:00080000:2.0:1789761818.363729:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.363733:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761818.363736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761818.363744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507551872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761818.363751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.363754:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761818.363755:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.363757:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1796 00000100:00080000:2.0:1789761818.363762:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.363765:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761818.363768:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761818.363775:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507552000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761818.363783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.363786:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761818.363791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761818.363792:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761818.363802:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1796 00000100:00080000:2.0:1789761818.363807:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.363811:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761818.363815:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761818.363826:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507552128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761818.363831:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761818.363834:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761818.363835:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761818.363838:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1796 00000100:00080000:2.0:1789761818.363843:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761818.363846:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761818.363851:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761818.363852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761818] req@ffff8d25c8020000 x1876699507551616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761818.363868:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507552256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761818.363882:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507551616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761818.363894:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761818.363898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761818.363905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507551616:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761818.363953:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761818.363975:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761818.363993:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761818] req@ffff8d25c8020700 x1876699507551744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761818.364002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507551744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761818.364008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761818.364010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761818.364012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761818.364016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761818.364029:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761818.364037:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761818] req@ffff8d25c8023100 x1876699507551872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761818.364044:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507551872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761818.364049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761818.364051:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761818.364053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507551872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761818.364057:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761818.364069:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761818.364075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761818] req@ffff8d25c8023b80 x1876699507552000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761818.364083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507552000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761818.364088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761818.364090:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761818.364092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761818.364095:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507552128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761818.364106:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761818.364112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761818] req@ffff8d25c8020380 x1876699507552128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761818.364119:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507552128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761818.364125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761818.364127:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761818.364129:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507552128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761818.364133:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507552256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761818.364143:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761818.364149:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761818/real 1789761818] req@ffff8d25c8022d80 x1876699507552256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761818.364156:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507552256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761843 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761818.364160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761818.364163:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761818.364165:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507552256:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761818.427534:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761819.451507:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761820.475533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761821.499532:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761822.523606:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761823.483506:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761823.483516:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761823.483520:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761823.483530:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1801 00000100:00080000:0.0:1789761823.483539:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761823.483546:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761823.483582:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507552384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761823.483614:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761823.483619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761823.483621:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761823.483625:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1801 00000100:00080000:0.0:1789761823.483630:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761823.483635:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761823.483645:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761823.483654:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507552384:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761823.483662:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507552512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761823.483675:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761823.483678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761823.483681:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761823.483685:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1801 00000100:00080000:0.0:1789761823.483700:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761823.483705:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761823.483711:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789761823.483719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761823.483727:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507552640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761823.483738:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761823.483742:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761823.483744:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761823.483748:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1801 00000100:00080000:0.0:1789761823.483753:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761823.483756:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761823/real 1789761823] req@ffff8d25d25bbb80 x1876699507552384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761823.483761:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761823.483777:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507552384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761823.483783:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1789761823.483786:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761823.483790:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761823.483794:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507552384:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761823.483806:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507552768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761823.483816:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552512:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761823.483823:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761823.483829:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761823.483831:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761823.483837:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761823.483838:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1801 00000100:00080000:0.0:1789761823.483846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761823.483850:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761823/real 1789761823] req@ffff8d25d25b8000 x1876699507552512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761823.483855:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761823.483868:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761823.483868:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507552512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761823.483879:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761823.483881:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761823.483884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761823.483889:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507552896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761823.483905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761823.483925:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761823.483930:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761823.483932:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789761823.483936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761823.483941:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1801 00000100:00080000:0.0:1789761823.483947:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789761823.483948:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761823/real 1789761823] req@ffff8d25d25ba300 x1876699507552640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761823.483956:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761823.483962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507552640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761823.483967:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789761823.483970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761823.483973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761823.483976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761823.483985:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507553024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761823.483990:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761823.484011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761823.484020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761823/real 1789761823] req@ffff8d25d25b8a80 x1876699507552768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761823.484029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507552768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761823.484034:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761823.484037:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761823.484039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507552768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761823.484042:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507552896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761823.484052:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761823.484057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761823/real 1789761823] req@ffff8d25d25b9180 x1876699507552896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761823.484069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507552896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761823.484074:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761823.484076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761823.484078:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507552896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761823.484085:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507553024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761823.484095:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761823.484101:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761823/real 1789761823] req@ffff8d25d25ba680 x1876699507553024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761823.484109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507553024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761848 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761823.484117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761823.484122:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761823.484126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507553024:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761823.547518:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761824.571587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761825.595557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761826.619577:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761827.643489:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761828.603498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761828.603506:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761828.603510:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761828.603520:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1806 00000100:00080000:3.0:1789761828.603528:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761828.603534:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761828.603565:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507553152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761828.603586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761828.603589:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761828.603590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761828.603593:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1806 00000100:00080000:3.0:1789761828.603595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761828.603598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761828.603605:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761828.603615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507553280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761828.603626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761828.603630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761828.603632:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761828.603637:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1806 00000100:00100000:2.0:1789761828.603642:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507553152:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761828.603643:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761828.603652:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761828.603658:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761828.603670:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507553408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761828.603678:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761828.603680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761828.603682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761828.603684:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1806 00000100:00080000:3.0:1789761828.603688:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761828.603691:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761828.603695:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761828.603703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507553536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761828.603710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761828.603712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761828.603713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761828.603715:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1806 00000100:00080000:3.0:1789761828.603718:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789761828.603722:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761828.603726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761828.603731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761828.603739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507553664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761828.603745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761828.603747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761828.603749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761828.603752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1806 00000100:00080000:3.0:1789761828.603754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761828.603756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761828.603759:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761828.603767:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761828/real 1789761828] req@ffff8d25d1b0bb80 x1876699507553152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761828.603771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507553792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789761828.603808:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507553152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761828.603822:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761828.603826:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761828.603830:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507553152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761828.603838:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507553280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761828.603876:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761828.603889:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761828/real 1789761828] req@ffff8d25d1b09880 x1876699507553280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761828.603903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507553280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761828.603912:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761828.603915:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761828.603919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507553280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761828.603946:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507553408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761828.603967:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761828.603979:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761828/real 1789761828] req@ffff8d25d1b0a680 x1876699507553408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761828.603991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507553408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761828.604000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761828.604003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761828.604006:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507553408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761828.604013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507553536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789761828.604032:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761828.604044:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761828/real 1789761828] req@ffff8d25d1b08e00 x1876699507553536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761828.604056:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507553536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761828.604064:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761828.604068:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761828.604071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507553536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789761828.604077:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507553664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761828.604095:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761828.604105:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761828/real 1789761828] req@ffff8d25d1b0b800 x1876699507553664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761828.604117:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507553664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761828.604125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761828.604128:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761828.604131:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507553664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761828.604138:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507553792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789761828.604154:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761828.604164:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761828/real 1789761828] req@ffff8d25d1b08700 x1876699507553792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761828.604185:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507553792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761853 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761828.604192:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761828.604195:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761828.604199:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507553792:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761828.667576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761829.691511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761830.715575:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761831.739516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761832.763555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761833.723523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761833.723534:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761833.723539:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761833.723550:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1811 00000100:00080000:2.0:1789761833.723562:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761833.723570:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761833.723614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507553920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761833.723644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761833.723647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761833.723648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761833.723652:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1811 00000100:00080000:2.0:1789761833.723654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761833.723657:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761833.723668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761833.723680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507554048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761833.723692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761833.723699:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507553920:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761833.723700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761833.723704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761833.723719:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1811 00000100:00080000:2.0:1789761833.723725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761833.723732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761833.723739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761833.723751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507554176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761833.723759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761833.723762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761833.723764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761833.723768:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1811 00000100:00080000:2.0:1789761833.723771:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789761833.723775:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761833.723786:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761833.723792:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761833.723806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507554304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761833.723826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761833.723830:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761833.723834:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789761833.723837:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761833/real 1789761833] req@ffff8d25c8022300 x1876699507553920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761833.723847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1811 00000100:00080000:2.0:1789761833.723860:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761833.723869:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507553920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761833.723871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761833.723888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789761833.723891:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761833.723897:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789761833.723905:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507554432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761833.723906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507553920:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761833.723928:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761833.723932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761833.723934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761833.723937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1811 00000100:00080000:2.0:1789761833.723941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761833.723944:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761833.723948:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761833.723954:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761833.723959:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507554560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761833.723991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761833.724009:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761833/real 1789761833] req@ffff8d25c8023480 x1876699507554048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761833.724019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507554048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761833.724026:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761833.724028:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761833.724031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761833.724035:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761833.724048:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761833.724055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761833/real 1789761833] req@ffff8d25c8020e00 x1876699507554176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761833.724068:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507554176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761833.724073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761833.724075:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761833.724077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761833.724081:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761833.724094:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761833.724100:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761833/real 1789761833] req@ffff8d25c8021500 x1876699507554304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761833.724109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507554304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761833.724114:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761833.724116:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761833.724118:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761833.724121:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507554432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761833.724141:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761833.724152:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761833/real 1789761833] req@ffff8d25c8021c00 x1876699507554432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761833.724159:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507554432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761833.724170:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761833.724172:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761833.724174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507554432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761833.724188:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507554560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761833.724200:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761833.724208:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761833/real 1789761833] req@ffff8d25c8021880 x1876699507554560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761833.724221:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507554560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761858 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761833.724229:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761833.724233:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761833.724237:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507554560:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761833.787565:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761834.811530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761835.835562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761836.859550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761837.883579:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761838.843446:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761838.843455:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761838.843460:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761838.843471:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1816 00000100:00080000:2.0:1789761838.843482:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761838.843490:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761838.843526:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507554688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761838.843572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761838.843576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761838.843579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761838.843584:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1816 00000100:00080000:2.0:1789761838.843588:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761838.843593:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761838.843604:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761838.843620:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507554816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761838.843623:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507554688:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761838.843640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761838.843644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761838.843646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761838.843650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1816 00000100:00080000:2.0:1789761838.843654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761838.843658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761838.843664:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761838.843677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507554944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761838.843687:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761838.843689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761838.843691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761838.843697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761838.843699:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1816 00000100:00080000:2.0:1789761838.843704:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761838.843708:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761838.843712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761838.843724:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507555072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761838.843735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761838.843738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761838.843742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761838.843747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1816 00000100:00080000:2.0:1789761838.843751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761838.843755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761838.843760:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761838.843771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507555200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761838.843789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761838.843792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761838.843793:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761838.843796:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1816 00000100:00080000:2.0:1789761838.843799:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761838.843802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761838.843805:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761838.843815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507555328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761838.843830:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761838/real 1789761838] req@ffff8d25c8021880 x1876699507554688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761838.843974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507554688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761838.843987:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761838.844055:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761838.844060:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507554688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761838.844085:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761838.844112:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761838.844127:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761838/real 1789761838] req@ffff8d25c8021c00 x1876699507554816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761838.844141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507554816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761838.844151:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761838.844154:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761838.844158:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761838.844164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761838.844195:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761838.844209:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761838/real 1789761838] req@ffff8d25c8021500 x1876699507554944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761838.844221:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507554944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761838.844236:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761838.844240:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761838.844243:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507554944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761838.844250:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761838.844270:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761838.844282:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761838/real 1789761838] req@ffff8d25c8020e00 x1876699507555072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761838.844294:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507555072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761838.844302:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761838.844306:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761838.844309:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761838.844315:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507555200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761838.844336:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761838.844348:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761838/real 1789761838] req@ffff8d25c8023480 x1876699507555200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761838.844360:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507555200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761838.844369:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761838.844372:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761838.844405:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507555200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761838.844416:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507555328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761838.844439:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761838.844451:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761838/real 1789761838] req@ffff8d25c8022300 x1876699507555328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761838.844463:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507555328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761863 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761838.844472:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761838.844475:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761838.844479:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507555328:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761838.907514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761839.931509:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761840.955540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761841.979607:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761843.003582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761843.963479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761843.963493:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761843.963498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761843.963510:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1821 00000100:00080000:2.0:1789761843.963522:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761843.963531:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761843.963580:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507555456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761843.963613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761843.963618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761843.963621:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761843.963627:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1821 00000100:00080000:2.0:1789761843.963632:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761843.963637:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761843.963646:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507555456:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789761843.963649:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761843.963667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507555584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761843.963680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761843.963684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761843.963687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761843.963691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1821 00000100:00080000:2.0:1789761843.963705:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761843.963710:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761843.963716:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761843.963722:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761843.963732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507555712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761843.963761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761843.963765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761843.963767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761843.963772:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1821 00000100:00080000:2.0:1789761843.963777:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761843.963785:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761843.963786:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761843/real 1789761843] req@ffff8d25c8022d80 x1876699507555456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761843.963793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761843.963810:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507555456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761843.963812:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507555840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761843.963849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761843.963849:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761843.963857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761843.963860:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761843.963862:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761843.963868:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507555456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761843.963869:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1821 00000100:00080000:2.0:1789761843.963876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761843.963884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761843.963890:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761843.963901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507555968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761843.963911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761843.963930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761843.963935:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761843.963937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761843.963944:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1821 00000800:00000100:3.0:1789761843.963945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761843.963952:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761843.963959:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761843.963963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761843/real 1789761843] req@ffff8d25c8020380 x1876699507555584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789761843.963966:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761843.963979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507555584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761843.963987:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507556096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761843.963988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761843.963995:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761843.964002:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761843.964015:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761843.964039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761843.964052:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761843/real 1789761843] req@ffff8d25c8023b80 x1876699507555712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761843.964066:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507555712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761843.964074:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761843.964076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761843.964079:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761843.964083:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761843.964100:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761843.964108:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761843/real 1789761843] req@ffff8d25c8023100 x1876699507555840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761843.964116:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507555840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761843.964122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761843.964124:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761843.964126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507555840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761843.964137:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507555968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761843.964148:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761843.964156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761843/real 1789761843] req@ffff8d25c8020700 x1876699507555968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761843.964169:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507555968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761843.964177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761843.964181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761843.964184:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507555968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761843.964190:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507556096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761843.964210:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761843.964218:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761843/real 1789761843] req@ffff8d25c8020000 x1876699507556096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761843.964226:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507556096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761868 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761843.964236:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761843.964238:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761843.964241:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507556096:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761844.027634:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761845.051572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761846.075599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761847.099536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761848.123596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761849.083511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761849.083527:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761849.083532:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761849.083548:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1826 00000100:00080000:0.0:1789761849.083560:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761849.083568:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761849.083612:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507556224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761849.083645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761849.083648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761849.083650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761849.083653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1826 00000100:00080000:0.0:1789761849.083656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761849.083659:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761849.083670:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761849.083680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507556352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761849.083688:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507556224:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761849.083691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761849.083697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761849.083699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761849.083716:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1826 00000100:00080000:0.0:1789761849.083718:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761849.083721:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761849.083725:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761849.083737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507556480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761849.083744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761849.083747:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761849.083748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761849.083751:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1826 00000100:00080000:0.0:1789761849.083754:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761849.083756:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761849.083760:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761849.083768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507556608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761849.083776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761849.083778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761849.083780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761849.083784:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761849.083787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1826 00000100:00080000:0.0:1789761849.083792:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761849.083797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761849.083803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761849.083815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507556736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761849.083826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761849.083831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761849.083833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761849.083835:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffff8d25d25bb800 x1876699507556224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761849.083841:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1826 00000100:00080000:0.0:1789761849.083848:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761849.083853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761849.083859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761849.083860:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507556224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761849.083871:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507556864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761849.083872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761849.083880:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761849.083888:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507556224:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761849.083912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507556352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761849.083930:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761849.083939:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffff8d25d25b9f80 x1876699507556352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761849.083947:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507556352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761849.083958:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761849.083960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761849.083963:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507556352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761849.083966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507556480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761849.083977:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761849.083983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffff8d25d25bad80 x1876699507556480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761849.083990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507556480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761849.083996:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761849.083998:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761849.084000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507556480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761849.084003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507556608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761849.084013:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761849.084020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffff8d25d25b8380 x1876699507556608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761849.084027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507556608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761849.084032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761849.084034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761849.084037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507556608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761849.084040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507556736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761849.084051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761849.084057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffff8d25d25b8700 x1876699507556736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761849.084064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507556736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761849.084069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761849.084071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761849.084073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507556736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761849.084076:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507556864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761849.084086:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761849.084092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761849/real 1789761849] req@ffff8d25d25bb100 x1876699507556864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761849.084099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507556864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761874 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761849.084104:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761849.084106:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761849.084108:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507556864:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761849.147547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761850.171585:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761851.195544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761852.219501:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761853.243467:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761854.203487:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761854.203499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761854.203506:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761854.203520:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1831 00000100:00080000:0.0:1789761854.203531:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761854.203540:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761854.203595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507556992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761854.203630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761854.203635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761854.203638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761854.203643:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1831 00000100:00080000:0.0:1789761854.203648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761854.203653:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761854.203689:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761854.203727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507557120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761854.203744:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761854.203750:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507556992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761854.203751:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761854.203757:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761854.203768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1831 00000100:00080000:0.0:1789761854.203772:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761854.203777:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761854.203783:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761854.203798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507557248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761854.203817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761854.203822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761854.203824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761854.203827:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1831 00000100:00080000:0.0:1789761854.203833:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761854.203839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761854.203844:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761854.203853:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761854.203869:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507557376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761854.203888:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761854.203897:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761854.203900:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761854.203906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffff8d25d25b9c00 x1876699507556992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761854.203910:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1831 00000100:00080000:0.0:1789761854.203925:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761854.203935:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761854.203940:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507556992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761854.203948:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761854.203957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761854.203963:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761854.203968:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507556992:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761854.203977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507557504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761854.203990:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761854.204010:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761854.204014:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761854.204018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557120:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761854.204026:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1831 00000100:00080000:0.0:1789761854.204033:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761854.204039:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761854.204050:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761854.204071:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761854.204081:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507557632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761854.204095:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffff8d25d25b8e00 x1876699507557120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761854.204125:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507557120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761854.204137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761854.204142:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761854.204147:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761854.204155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761854.204175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761854.204187:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffff8d25d25bb480 x1876699507557248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761854.204201:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507557248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761854.204211:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761854.204216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761854.204234:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761854.204245:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761854.204283:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761854.204305:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffff8d25d25bb100 x1876699507557376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761854.204328:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507557376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761854.204343:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761854.204349:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761854.204355:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761854.204365:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507557504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761854.204433:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761854.204447:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffff8d25d25b8700 x1876699507557504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761854.204462:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507557504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761854.204472:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761854.204497:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761854.204501:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507557504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761854.204526:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507557632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761854.204550:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761854.204562:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761854/real 1789761854] req@ffff8d25d25b8380 x1876699507557632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761854.204577:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507557632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761879 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761854.204587:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761854.204590:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761854.204593:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507557632:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761854.267579:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761855.291545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761856.315558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761857.339530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761858.363650:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761859.323496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.323505:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761859.323510:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.323519:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1837 00000100:00080000:2.0:1789761859.323527:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.323533:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761859.323566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507557760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761859.323591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.323594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761859.323596:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.323600:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1837 00000100:00080000:2.0:1789761859.323602:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.323605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761859.323613:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761859.323642:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507557888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761859.323657:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507557760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761859.323658:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.323670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761859.323674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.323681:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1837 00000100:00080000:2.0:1789761859.323686:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.323691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761859.323696:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761859.323709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507558016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761859.323717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.323720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761859.323723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.323725:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1837 00000100:00080000:2.0:1789761859.323729:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.323731:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789761859.323736:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789761859.323737:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761859.323748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507558144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761859.323757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.323760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761859.323761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761859.323764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1837 00000100:00080000:2.0:1789761859.323767:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.323770:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761859.323774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761859.323781:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffff8d25c8023800 x1876699507557760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761859.323785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507558272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761859.323812:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761859.323815:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761859.323817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761859.323821:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507557760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761859.323824:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1837 00000100:00080000:2.0:1789761859.323829:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761859.323835:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761859.323841:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761859.323856:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789761859.323861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507558400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761859.323863:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761859.323872:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507557760:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761859.323900:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761859.323924:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761859.323939:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffff8d25c8022a00 x1876699507557888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761859.323954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507557888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761859.323965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761859.323969:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761859.323974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507557888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761859.323981:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761859.324009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761859.324019:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffff8d25c8022680 x1876699507558016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761859.324028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507558016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761859.324035:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761859.324037:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761859.324039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761859.324044:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761859.324057:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761859.324064:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffff8d25c8021180 x1876699507558144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761859.324080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507558144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761859.324086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761859.324089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761859.324091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761859.324095:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507558272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761859.324107:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761859.324114:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffff8d25c8020a80 x1876699507558272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761859.324122:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507558272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761859.324127:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761859.324129:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761859.324131:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507558272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761859.324135:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507558400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761859.324153:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761859.324161:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761859/real 1789761859] req@ffff8d25c8020000 x1876699507558400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761859.324170:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507558400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761884 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761859.324175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761859.324178:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761859.324182:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507558400:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761859.387553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761860.411523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761861.435570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761862.459634:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761863.483663:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761864.443534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761864.443544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761864.443549:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761864.443560:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1842 00000100:00080000:0.0:1789761864.443569:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761864.443576:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761864.443621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507558528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761864.443640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761864.443655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761864.443657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761864.443660:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1842 00000100:00080000:0.0:1789761864.443663:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761864.443667:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761864.443675:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761864.443684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507558656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761864.443691:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507558528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761864.443695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761864.443699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761864.443700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761864.443704:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1842 00000100:00080000:0.0:1789761864.443706:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761864.443708:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761864.443712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761864.443733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507558784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761864.443740:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761864.443743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761864.443745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761864.443748:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1842 00000100:00080000:0.0:1789761864.443750:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761864.443752:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761864.443755:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761864.443763:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507558912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761864.443769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761864.443772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761864.443773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761864.443776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1842 00000100:00080000:0.0:1789761864.443778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761864.443781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761864.443785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761864.443794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507559040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761864.443797:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761864.443807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761864.443810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761864.443812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761864.443815:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1842 00000100:00080000:0.0:1789761864.443818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761864.443822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761864.443825:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761864.443833:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507559168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761864.443844:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761864.443872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761864.443893:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761864.443906:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559168:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761864.507534:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761865.531492:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761866.555525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789761867.515785:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789761867.515833:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789761867.515849:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761867.515883:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761867.515893:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761867.515903:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761867.515916:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789761867.515927:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761867.515933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761864/real 1789761867] req@ffff8d25d25b9f80 x1876699507558656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761867.515961:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507558656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761867.515981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761867.515986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761867.515991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761867.516000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761864/real 1789761867] req@ffff8d25d25bb800 x1876699507558784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761867.516012:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507558784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761867.516020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761867.516024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761867.516028:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761867.516035:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761864/real 1789761867] req@ffff8d25d25ba680 x1876699507558912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761867.516047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507558912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761867.516053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761867.516057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761867.516060:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507558912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761867.516066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761864/real 1789761867] req@ffff8d25d25bad80 x1876699507558528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761867.516077:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507558528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761867.516085:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761867.516088:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761867.516092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507558528:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761867.516117:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761864/real 1789761867] req@ffff8d25d25b9180 x1876699507559040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761867.516131:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507559040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761867.516140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761867.516144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761867.516147:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761867.516154:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761864/real 1789761867] req@ffff8d25d25b8a80 x1876699507559168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761867.516168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507559168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761889 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761867.516177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761867.516181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761867.516185:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559168:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761867.579547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761868.603538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761869.563491:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761869.563500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761869.563505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761869.563513:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1847 00000100:00080000:3.0:1789761869.563523:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761869.563529:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761869.563577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507559296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761869.563602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761869.563605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761869.563606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761869.563610:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1847 00000100:00080000:3.0:1789761869.563612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761869.563616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761869.563623:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761869.563633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507559424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761869.563640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761869.563643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761869.563644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761869.563646:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1847 00000100:00080000:3.0:1789761869.563648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761869.563651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761869.563655:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761869.563662:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507559296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761869.563663:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507559552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761869.563688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761869.563691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761869.563693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761869.563699:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1847 00000100:00080000:3.0:1789761869.563702:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761869.563706:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761869.563712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761869.563725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507559680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761869.563735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761869.563738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761869.563740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789761869.563746:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761869.563750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1847 00000100:00080000:3.0:1789761869.563757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761869.563762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761869.563768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761869.563782:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507559808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761869.563792:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789761869.563797:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761869/real 1789761869] req@ffff8d25d1b08700 x1876699507559296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761869.563797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761869.563802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761869.563811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1847 00000100:00080000:3.0:1789761869.563819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761869.563828:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761869.563836:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761869.563843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507559296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761869.563851:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507559936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761869.563859:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761869.563868:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761869.563875:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507559296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761869.563885:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507559424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761869.563908:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761869.563922:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761869/real 1789761869] req@ffff8d25d1b0b800 x1876699507559424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761869.563935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507559424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761869.563945:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761869.563948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761869.563952:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507559424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761869.563979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507559552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761869.563998:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761869.564009:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761869/real 1789761869] req@ffff8d25d1b08e00 x1876699507559552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761869.564022:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507559552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761869.564036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761869.564039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761869.564043:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507559552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761869.564049:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507559680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761869.564068:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761869.564079:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761869/real 1789761869] req@ffff8d25d1b0a680 x1876699507559680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761869.564091:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507559680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761869.564100:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761869.564103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761869.564107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507559680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761869.564113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761869.564132:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761869.564142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761869/real 1789761869] req@ffff8d25d1b09880 x1876699507559808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761869.564154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507559808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761869.564162:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761869.564165:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761869.564169:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761869.564176:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761869.564193:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761869.564203:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761869/real 1789761869] req@ffff8d25d1b0bb80 x1876699507559936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761869.564215:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507559936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761894 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761869.564222:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761869.564225:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761869.564229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507559936:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761869.627570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761870.651518:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761871.675564:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761872.699564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761873.723597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761874.683544:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761874.683552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761874.683556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761874.683566:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1852 00000100:00080000:1.0:1789761874.683575:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761874.683583:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761874.683619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507560064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761874.683641:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761874.683645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761874.683647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761874.683652:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1852 00000100:00080000:1.0:1789761874.683657:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761874.683661:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761874.683671:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761874.683681:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507560064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761874.683686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507560192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761874.683696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761874.683700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761874.683702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761874.683707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1852 00000100:00080000:1.0:1789761874.683710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761874.683713:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761874.683717:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761874.683727:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507560320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761874.683736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761874.683740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789761874.683740:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761874.683742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761874.683746:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1852 00000100:00080000:1.0:1789761874.683749:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761874.683752:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761874.683756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761874.683766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507560448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761874.683778:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761874/real 1789761874] req@ffff8d25c8168000 x1876699507560064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761874.683783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761874.683787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761874.683800:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761874.683803:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1852 00000100:00080000:1.0:1789761874.683807:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761874.683811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761874.683816:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761874.683827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507560576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761874.683836:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761874.683839:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761874.683841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761874.683845:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1852 00000100:00080000:1.0:1789761874.683849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761874.683853:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761874.683859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761874.683870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507560704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761874.683883:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507560064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761874.683895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761874.683899:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761874.683904:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507560064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761874.683926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761874.683947:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761874.683957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761874/real 1789761874] req@ffff8d25c8168380 x1876699507560192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761874.683966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507560192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761874.683977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761874.683979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761874.683981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761874.683985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761874.683996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761874.684002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761874/real 1789761874] req@ffff8d25c816b480 x1876699507560320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761874.684010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507560320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761874.684015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761874.684016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761874.684018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761874.684032:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761874.684042:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761874.684048:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761874/real 1789761874] req@ffff8d25c816a300 x1876699507560448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761874.684055:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507560448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761874.684060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761874.684062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761874.684065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761874.684068:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507560576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761874.684182:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761874.684195:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761874/real 1789761874] req@ffff8d25c816ad80 x1876699507560576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761874.684208:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507560576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761874.684216:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761874.684266:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761874.684271:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507560576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761874.684278:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507560704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761874.684301:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761874.684310:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761874/real 1789761874] req@ffff8d25c8168a80 x1876699507560704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761874.684318:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507560704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761899 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761874.684324:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761874.684325:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761874.684327:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507560704:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761874.747533:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761875.771593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761876.795533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761877.819600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761878.843600:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761879.803532:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761879.803542:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761879.803548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761879.803558:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1857 00000100:00080000:1.0:1789761879.803576:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761879.803594:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761879.803629:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507560832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761879.803653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761879.803657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761879.803659:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761879.803664:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1857 00000100:00080000:1.0:1789761879.803667:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761879.803672:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761879.803681:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761879.803694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507560832:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761879.803696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507560960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761879.803710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761879.803713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761879.803716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761879.803720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1857 00000100:00080000:1.0:1789761879.803723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761879.803727:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761879.803744:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761879.803755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507561088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761879.803765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761879.803770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789761879.803773:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761879.803774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761879.803781:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1857 00000100:00080000:1.0:1789761879.803786:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761879.803790:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761879.803795:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761879.803817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507561216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761879.803828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761879.803830:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761879/real 1789761879] req@ffff8d25c8169c00 x1876699507560832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761879.803834:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761879.803838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761879.803847:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1857 00000100:00080000:1.0:1789761879.803853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761879.803857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761879.803862:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507560832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789761879.803874:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761879.803887:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789761879.803890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507561344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761879.803895:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761879.803905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507560832:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761879.803909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761879.803914:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761879.803916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761879.803921:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1857 00000100:00080000:1.0:1789761879.803924:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761879.803937:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761879.803942:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761879.803955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761879.803956:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507561472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761879.803991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761879.804017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761879/real 1789761879] req@ffff8d25c8169880 x1876699507560960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761879.804031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507560960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761879.804040:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761879.804044:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761879.804047:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507560960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761879.804054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761879.804104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761879.804119:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761879/real 1789761879] req@ffff8d25c8168a80 x1876699507561088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761879.804137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507561088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761879.804160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761879.804165:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761879.804170:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761879.804179:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761879.804222:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761879.804241:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761879/real 1789761879] req@ffff8d25c816ad80 x1876699507561216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761879.804254:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507561216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761879.804262:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761879.804266:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761879.804269:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761879.804274:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507561344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761879.804327:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761879.804343:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761879/real 1789761879] req@ffff8d25c816a300 x1876699507561344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761879.804383:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507561344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761879.804439:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761879.804447:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761879.804453:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507561344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761879.804543:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507561472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761879.804567:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761879.804580:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761879/real 1789761879] req@ffff8d25c816b480 x1876699507561472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761879.804592:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507561472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761904 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761879.804613:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761879.804617:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761879.804620:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507561472:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761879.867631:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761880.891629:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761881.915580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761882.939551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761883.963540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761884.923503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761884.923512:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761884.923515:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761884.923525:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1862 00000100:00080000:0.0:1789761884.923533:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761884.923540:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761884.923575:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507561600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761884.923602:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761884.923607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761884.923609:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761884.923614:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1862 00000100:00080000:0.0:1789761884.923617:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761884.923621:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761884.923630:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761884.923641:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507561728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761884.923649:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507561600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761884.923652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761884.923658:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761884.923661:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761884.923665:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1862 00000100:00080000:0.0:1789761884.923668:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761884.923670:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761884.923672:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761884.923682:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507561856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761884.923699:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761884.923701:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761884.923703:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761884.923706:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1862 00000100:00080000:0.0:1789761884.923708:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761884.923711:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761884.923715:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761884.923723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507561984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761884.923729:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761884.923732:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761884.923734:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761884.923740:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1862 00000800:00000100:3.0:1789761884.923740:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761884.923745:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761884.923748:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761884.923751:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761884.923759:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507562112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761884.923765:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761884.923768:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761884.923770:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761884.923773:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1862 00000100:00080000:0.0:1789761884.923775:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761884.923777:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761884.923781:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761884.923793:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507562240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761884.923793:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761884/real 1789761884] req@ffff8d25d25ba300 x1876699507561600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761884.923826:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507561600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761884.923855:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761884.923859:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761884.923863:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507561600:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761884.923886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761884.923911:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761884.923925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761884/real 1789761884] req@ffff8d25d25b8000 x1876699507561728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761884.923937:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507561728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761884.923947:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761884.923951:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761884.923954:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761884.923961:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761884.923981:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761884.923992:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761884/real 1789761884] req@ffff8d25d25bbb80 x1876699507561856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761884.924005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507561856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761884.924023:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761884.924026:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761884.924030:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761884.924037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761884.924056:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761884.924066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761884/real 1789761884] req@ffff8d25d25b8a80 x1876699507561984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761884.924077:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507561984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761884.924084:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761884.924087:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761884.924090:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507561984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761884.924096:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507562112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761884.924119:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761884.924128:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761884/real 1789761884] req@ffff8d25d25b9180 x1876699507562112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761884.924139:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507562112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761884.924147:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761884.924150:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761884.924153:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507562112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761884.924160:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507562240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761884.924181:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761884.924195:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761884/real 1789761884] req@ffff8d25d25bad80 x1876699507562240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761884.924207:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507562240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761909 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761884.924215:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761884.924218:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761884.924221:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507562240:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761884.987559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761886.011504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761887.035540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761888.059498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761889.083502:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761890.043484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.043492:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761890.043497:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.043506:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1867 00000100:00080000:0.0:1789761890.043514:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.043520:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761890.043554:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507562368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761890.043576:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.043579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761890.043580:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.043583:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1867 00000100:00080000:0.0:1789761890.043585:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.043588:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761890.043595:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761890.043603:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507562496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761890.043611:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.043613:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761890.043615:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.043618:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1867 00000100:00080000:0.0:1789761890.043620:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.043622:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761890.043626:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761890.043630:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507562368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761890.043636:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507562624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761890.043644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.043647:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761890.043648:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.043651:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1867 00000100:00080000:0.0:1789761890.043653:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.043655:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761890.043659:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761890.043667:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507562752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761890.043674:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.043677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761890.043678:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.043680:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1867 00000100:00080000:0.0:1789761890.043683:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761890.043686:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761890.043687:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761890.043692:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761890.043701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507562880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761890.043707:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761890.043710:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761890.043712:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761890.043715:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1867 00000100:00080000:0.0:1789761890.043718:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761890.043720:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761890.043724:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761890.043725:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffff8d25d25ba680 x1876699507562368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761890.043735:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507563008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761890.043743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507562368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761890.043767:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761890.043771:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761890.043776:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507562368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761890.043785:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507562496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761890.043809:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761890.043822:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffff8d25d25bb800 x1876699507562496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761890.043841:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507562496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761890.043848:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761890.043850:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761890.043853:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507562496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761890.043872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507562624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761890.043887:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761890.043895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffff8d25d25b9f80 x1876699507562624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761890.043903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507562624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761890.043908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761890.043911:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761890.043913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507562624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761890.043917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507562752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761890.043927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761890.043932:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffff8d25d25b8380 x1876699507562752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761890.043940:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507562752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761890.043953:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761890.043956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761890.043960:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507562752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761890.043966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507562880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761890.043985:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761890.043997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffff8d25d25b8700 x1876699507562880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761890.044008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507562880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761890.044014:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761890.044016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761890.044018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507562880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761890.044022:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507563008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761890.044040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761890.044052:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761890/real 1789761890] req@ffff8d25d25bb100 x1876699507563008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761890.044065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507563008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761915 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761890.044073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761890.044076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761890.044080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507563008:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761890.107510:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761891.131522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761892.155534:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761893.179531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761894.203547:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761895.163468:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761895.163479:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761895.163483:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761895.163494:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1872 00000100:00080000:0.0:1789761895.163504:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761895.163511:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761895.163544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507563136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761895.163575:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761895.163579:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761895.163581:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761895.163586:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1872 00000100:00080000:0.0:1789761895.163590:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761895.163594:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761895.163603:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761895.163616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507563264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761895.163624:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507563136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761895.163633:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761895.163637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761895.163639:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761895.163646:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1872 00000100:00080000:0.0:1789761895.163649:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761895.163656:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761895.163662:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761895.163674:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507563392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761895.163682:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761895.163696:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761895.163700:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761895.163702:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761895.163707:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1872 00000100:00080000:0.0:1789761895.163712:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761895.163716:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761895.163719:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffff8d25d25bb480 x1876699507563136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789761895.163723:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761895.163745:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507563136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761895.163749:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507563520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761895.163761:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761895.163769:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761895.163770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761895.163774:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507563136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761895.163778:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761895.163781:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761895.163786:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1872 00000100:00080000:0.0:1789761895.163789:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761895.163794:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761895.163797:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507563264:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789761895.163802:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761895.163818:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507563648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761895.163824:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761895.163837:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761895.163840:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffff8d25d25b8e00 x1876699507563264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761895.163853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761895.163858:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761895.163859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507563264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761895.163866:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1872 00000100:00080000:3.0:1789761895.163872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761895.163874:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761895.163878:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761895.163882:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761895.163883:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507563264:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789761895.163888:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761895.163893:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507563392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761895.163906:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507563776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761895.163913:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761895.163925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffff8d25d25b9c00 x1876699507563392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761895.163934:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507563392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761895.163941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761895.163943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761895.163945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507563392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761895.163951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507563520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761895.163963:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761895.163971:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffff8d25d25bb100 x1876699507563520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761895.163978:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507563520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761895.163984:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761895.163985:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761895.163987:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507563520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761895.163998:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507563648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761895.164009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761895.164015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffff8d25d25b8700 x1876699507563648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761895.164023:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507563648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761895.164028:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761895.164030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761895.164032:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507563648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761895.164035:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507563776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761895.164045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761895.164050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761895/real 1789761895] req@ffff8d25d25b8380 x1876699507563776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761895.164058:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507563776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761920 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761895.164062:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761895.164064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761895.164066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507563776:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761895.227528:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761896.251526:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761897.275508:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761898.299568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761899.323516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761900.283494:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761900.283504:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761900.283508:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761900.283518:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1877 00000100:00080000:2.0:1789761900.283527:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761900.283534:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761900.283582:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507563904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761900.283607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761900.283611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761900.283613:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761900.283617:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1877 00000100:00080000:2.0:1789761900.283621:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761900.283626:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761900.283635:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761900.283648:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507564032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761900.283655:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507563904:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761900.283673:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761900.283677:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761900.283679:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761900.283684:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1877 00000100:00080000:2.0:1789761900.283688:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761900.283692:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761900.283697:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761900.283709:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507564160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761900.283719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761900.283722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761900.283727:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761900.283730:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761900.283735:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1877 00000100:00080000:2.0:1789761900.283739:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761900.283743:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761900.283749:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761900.283760:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507564288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761900.283763:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffff8d25c8020700 x1876699507563904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761900.283787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761900.283795:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761900.283799:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761900.283806:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1877 00000100:00100000:3.0:1789761900.283810:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507563904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761900.283812:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761900.283820:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761900.283831:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761900.283831:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761900.283838:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761900.283842:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507563904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761900.283848:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507564416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761900.283859:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761900.283864:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761900.283864:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564032:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789761900.283867:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761900.283874:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1877 00000100:00080000:2.0:1789761900.283878:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761900.283882:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761900.283888:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761900.283895:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761900.283902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507564544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761900.283923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffff8d25c8023100 x1876699507564032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761900.283939:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507564032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761900.283946:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761900.283949:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761900.283951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761900.283955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761900.283972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761900.283980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffff8d25c8023b80 x1876699507564160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761900.283988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507564160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761900.283997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761900.284000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761900.284009:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761900.284013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761900.284026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761900.284032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffff8d25c8020380 x1876699507564288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761900.284040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507564288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761900.284045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761900.284047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761900.284049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761900.284053:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507564416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761900.284063:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761900.284069:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffff8d25c8022d80 x1876699507564416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761900.284081:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507564416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761900.284086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761900.284088:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761900.284091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507564416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761900.284103:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507564544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761900.284115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761900.284122:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761900/real 1789761900] req@ffff8d25c8022300 x1876699507564544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761900.284130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507564544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761925 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761900.284135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761900.284136:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761900.284138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507564544:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761900.347565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761901.371516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761902.395520:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761903.419499:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761904.443628:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761905.403553:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761905.403569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761905.403575:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761905.403591:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1883 00000100:00080000:1.0:1789761905.403605:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761905.403616:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761905.403665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507564672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761905.403696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761905.403700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761905.403702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761905.403707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1883 00000100:00080000:1.0:1789761905.403712:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761905.403717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761905.403729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761905.403738:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507564672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761905.403745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507564800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761905.403754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761905.403757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761905.403758:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761905.403762:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1883 00000100:00080000:1.0:1789761905.403765:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761905.403768:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761905.403773:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761905.403783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507564928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761905.403790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761905.403793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761905.403794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761905.403797:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1883 00000100:00080000:1.0:1789761905.403799:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761905.403802:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761905.403805:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761905.403811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507565056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761905.403818:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761905.403822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761905.403825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789761905.403825:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761905.403832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1883 00000100:00080000:1.0:1789761905.403834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761905.403837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761905.403840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761905.403848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507565184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761905.403859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761905.403862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761905.403863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761905.403866:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1883 00000100:00080000:1.0:1789761905.403869:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761905.403872:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761905.403876:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761905.403877:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761905/real 1789761905] req@ffff8d25c8168380 x1876699507564672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761905.403886:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507565312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761905.403899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507564672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761905.403910:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761905.403914:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761905.403920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507564672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761905.403952:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761905.403981:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761905.403993:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761905/real 1789761905] req@ffff8d25c8168000 x1876699507564800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761905.404002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507564800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761905.404008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761905.404011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761905.404013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761905.404017:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761905.404030:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761905.404037:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761905/real 1789761905] req@ffff8d25c8169500 x1876699507564928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761905.404045:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507564928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761905.404051:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761905.404053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761905.404055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507564928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761905.404058:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761905.404069:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761905.404075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761905/real 1789761905] req@ffff8d25c8168700 x1876699507565056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761905.404084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507565056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761905.404089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761905.404091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761905.404093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761905.404096:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507565184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761905.404107:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761905.404114:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761905/real 1789761905] req@ffff8d25c8169f80 x1876699507565184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761905.404122:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507565184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761905.404127:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761905.404128:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761905.404130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507565184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761905.404133:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507565312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761905.404143:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761905.404149:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761905/real 1789761905] req@ffff8d25c8168e00 x1876699507565312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761905.404157:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507565312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761930 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761905.404166:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761905.404169:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761905.404171:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507565312:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761905.467581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761906.491571:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761907.515608:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761908.539579:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761909.563565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761910.523541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761910.523549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761910.523554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761910.523567:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1888 00000100:00080000:2.0:1789761910.523578:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761910.523586:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761910.523628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507565440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761910.523664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761910.523667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761910.523668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761910.523672:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1888 00000100:00080000:2.0:1789761910.523676:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761910.523679:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761910.523690:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761910.523701:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507565568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761910.523710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761910.523714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761910.523717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761910.523725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1888 00000100:00100000:3.0:1789761910.523727:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507565440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761910.523729:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761910.523735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761910.523745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761910.523757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507565696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761910.523764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761910.523778:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761910.523779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761910.523782:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1888 00000100:00080000:2.0:1789761910.523786:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761910.523790:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761910.523800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761910.523809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507565824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761910.523817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761910.523819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761910.523820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761910.523827:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1888 00000800:00000100:3.0:1789761910.523827:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761910.523839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761910.523843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761910.523846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761910.523853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507565952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761910.523859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761910.523862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761910.523863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761910.523865:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1888 00000100:00080000:2.0:1789761910.523867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761910.523869:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761910.523871:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761910.523878:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507566080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761910.524001:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761910/real 1789761910] req@ffff8d25c8023480 x1876699507565440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761910.524126:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507565440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761910.524135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761910.524139:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761910.524142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507565440:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761910.524258:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761910.524291:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761910.524305:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761910/real 1789761910] req@ffff8d25c8020e00 x1876699507565568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761910.524315:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507565568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761910.524326:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761910.524328:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761910.524330:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761910.524335:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761910.524349:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761910.524356:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761910/real 1789761910] req@ffff8d25c8021500 x1876699507565696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761910.524364:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507565696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761910.524369:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761910.524371:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761910.524373:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761910.524411:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761910.524433:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761910.524441:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761910/real 1789761910] req@ffff8d25c8021c00 x1876699507565824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761910.524449:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507565824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761910.524454:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761910.524457:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761910.524459:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507565824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761910.524463:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507565952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761910.524474:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761910.524481:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761910/real 1789761910] req@ffff8d25c8021880 x1876699507565952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761910.524488:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507565952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761910.524493:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761910.524495:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761910.524497:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507565952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761910.524501:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507566080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761910.524511:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761910.524518:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761910/real 1789761910] req@ffff8d25c8022300 x1876699507566080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761910.524525:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507566080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761935 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761910.524530:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761910.524532:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761910.524534:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507566080:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761910.587576:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761911.611536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761912.635565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761913.659653:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761914.683610:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761915.643513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761915.643524:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761915.643529:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761915.643541:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1893 00000100:00080000:2.0:1789761915.643551:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761915.643559:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761915.643602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507566208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761915.643629:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761915.643632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761915.643634:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761915.643639:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1893 00000100:00080000:2.0:1789761915.643644:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761915.643649:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761915.643661:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761915.643683:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507566208:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761915.643684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507566336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761915.643699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761915.643702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761915.643704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761915.643707:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1893 00000100:00080000:2.0:1789761915.643711:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761915.643714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761915.643719:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761915.643730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507566464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761915.643737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761915.643740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761915.643741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761915.643744:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1893 00000100:00080000:2.0:1789761915.643748:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761915.643751:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761915.643754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761915.643762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507566592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761915.643766:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761915.643776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761915.643780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761915.643781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761915.643785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1893 00000100:00080000:2.0:1789761915.643788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761915.643793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761915.643798:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761915.643814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507566720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761915.643817:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761915/real 1789761915] req@ffff8d25c8022d80 x1876699507566208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761915.643845:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761915.643848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761915.643851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507566208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761915.643852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761915.643862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1893 00000100:00080000:2.0:1789761915.643868:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761915.643872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761915.643873:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761915.643881:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789761915.643882:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761915.643887:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507566208:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761915.643896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507566848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761915.643922:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507566336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761915.643950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761915.643963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761915/real 1789761915] req@ffff8d25c8020380 x1876699507566336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761915.643972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507566336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761915.643979:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761915.643981:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761915.643984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507566336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761915.643987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507566464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761915.644002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761915.644009:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761915/real 1789761915] req@ffff8d25c8023b80 x1876699507566464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761915.644017:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507566464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761915.644023:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761915.644024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761915.644026:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507566464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761915.644030:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507566592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761915.644041:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761915.644047:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761915/real 1789761915] req@ffff8d25c8023100 x1876699507566592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761915.644055:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507566592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761915.644060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761915.644062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761915.644064:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507566592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761915.644067:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507566720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761915.644078:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761915.644084:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761915/real 1789761915] req@ffff8d25c8020700 x1876699507566720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761915.644091:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507566720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761915.644097:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761915.644099:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761915.644101:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507566720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761915.644104:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507566848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761915.644114:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761915.644119:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761915/real 1789761915] req@ffff8d25c8020000 x1876699507566848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761915.644126:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507566848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761940 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761915.644131:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761915.644134:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761915.644135:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507566848:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761915.707552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761916.731526:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761917.755593:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761918.779527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761919.803681:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761920.763524:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761920.763538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761920.763543:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761920.763560:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1898 00000100:00080000:2.0:1789761920.763573:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761920.763581:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761920.763648:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507566976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761920.763681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761920.763700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761920.763703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761920.763708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1898 00000100:00080000:2.0:1789761920.763712:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761920.763718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761920.763729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761920.763737:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507566976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761920.763747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507567104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761920.763766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761920.763769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761920.763771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761920.763775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1898 00000100:00080000:2.0:1789761920.763790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761920.763794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761920.763799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761920.763811:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507567232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761920.763819:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761920.763829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789761920.763830:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761920.763837:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761920.763846:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1898 00000100:00080000:2.0:1789761920.763851:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761920.763856:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761920.763861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761920.763872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507567360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761920.763910:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761920.763915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761920.763922:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761920.763924:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761920/real 1789761920] req@ffff8d25c8020a80 x1876699507566976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761920.763936:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1898 00000100:00080000:2.0:1789761920.763943:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761920.763946:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761920.763951:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761920.763975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507567488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761920.763982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507566976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761920.764021:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761920.764033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761920.764042:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761920.764058:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1898 00000100:00080000:3.0:1789761920.764062:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761920.764065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761920.764072:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761920.764073:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761920.764086:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761920.764087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507566976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761920.764110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507567616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761920.764128:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761920.764158:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761920.764205:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761920/real 1789761920] req@ffff8d25c8021180 x1876699507567104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761920.764224:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507567104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761920.764231:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761920.764233:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761920.764236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761920.764240:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761920.764255:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761920.764262:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761920/real 1789761920] req@ffff8d25c8022680 x1876699507567232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761920.764270:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507567232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761920.764275:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761920.764277:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761920.764279:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761920.764284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761920.764295:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761920.764301:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761920/real 1789761920] req@ffff8d25c8022a00 x1876699507567360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761920.764309:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507567360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761920.764314:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761920.764316:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761920.764318:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761920.764321:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507567488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761920.764342:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761920.764349:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761920/real 1789761920] req@ffff8d25c8023800 x1876699507567488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761920.764356:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507567488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761920.764361:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761920.764363:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761920.764364:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507567488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761920.764376:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507567616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761920.764386:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761920.764422:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761920/real 1789761920] req@ffff8d25c8020000 x1876699507567616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761920.764430:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507567616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761945 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761920.764436:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761920.764438:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761920.764440:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507567616:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761920.827609:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761921.851573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761922.875515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761923.899576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761924.923542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761925.883496:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761925.883506:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761925.883511:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761925.883521:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1903 00000100:00080000:0.0:1789761925.883530:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761925.883537:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761925.883572:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507567744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761925.883599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761925.883607:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761925.883610:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761925.883617:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1903 00000100:00080000:0.0:1789761925.883621:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761925.883627:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761925.883632:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507567744:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789761925.883638:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761925.883658:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507567872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761925.883670:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761925.883674:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761925.883677:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761925.883682:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1903 00000100:00080000:0.0:1789761925.883686:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761925.883691:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761925.883696:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761925.883702:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761925.883715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507568000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761925.883728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761925.883732:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761925/real 1789761925] req@ffff8d25d25b9f80 x1876699507567744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761925.883734:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761925.883739:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761925.883749:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1903 00000100:00080000:0.0:1789761925.883753:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761925.883754:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507567744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761925.883762:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761925.883768:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789761925.883770:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:3.0:1789761925.883774:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761925.883779:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507567744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761925.883788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507568128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761925.883803:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761925.883804:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567872:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761925.883814:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761925.883816:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761925.883822:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1903 00000100:00080000:0.0:1789761925.883828:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761925.883831:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761925.883836:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761925.883843:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761925.883846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761925/real 1789761925] req@ffff8d25d25bb800 x1876699507567872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761925.883859:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507568256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761925.883889:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761925.883895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507567872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761925.883907:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761925.883912:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761925.883914:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761925.883919:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1903 00000100:00080000:3.0:1789761925.883920:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789761925.883927:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761925.883930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507567872:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761925.883935:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761925.883941:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761925.883943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761925.883955:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507568384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761925.883976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761925.883990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761925/real 1789761925] req@ffff8d25d25ba680 x1876699507568000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761925.883999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507568000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761925.884004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761925.884007:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761925.884009:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761925.884019:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761925.884033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761925.884039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761925/real 1789761925] req@ffff8d25d25bad80 x1876699507568128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761925.884047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507568128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761925.884056:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761925.884058:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761925.884061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761925.884064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507568256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761925.884074:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761925.884089:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761925/real 1789761925] req@ffff8d25d25b9180 x1876699507568256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761925.884096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507568256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761925.884101:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761925.884103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761925.884105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507568256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761925.884108:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507568384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761925.884118:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761925.884124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761925/real 1789761925] req@ffff8d25d25b8a80 x1876699507568384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761925.884152:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507568384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761950 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761925.884158:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761925.884160:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761925.884162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507568384:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761925.947595:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761926.971559:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761927.995636:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761929.019498:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761930.043577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761931.003498:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761931.003511:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761931.003517:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761931.003531:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1908 00000100:00080000:0.0:1789761931.003543:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761931.003553:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761931.003606:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507568512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761931.003634:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761931.003637:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761931.003639:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761931.003642:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1908 00000100:00080000:0.0:1789761931.003646:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761931.003650:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761931.003660:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761931.003672:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507568640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761931.003683:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761931.003685:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507568512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761931.003686:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761931.003696:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761931.003701:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1908 00000100:00080000:0.0:1789761931.003704:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761931.003707:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761931.003712:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761931.003723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507568768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761931.003735:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761931.003737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761931.003738:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761931.003742:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1908 00000100:00080000:0.0:1789761931.003745:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761931.003748:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761931.003752:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761931.003762:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507568896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761931.003770:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761931.003773:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761931.003774:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761931.003776:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1908 00000100:00080000:0.0:1789761931.003780:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761931.003783:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761931.003786:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761931.003796:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507569024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761931.003802:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761931.003805:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761931.003808:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761931.003813:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789761931.003816:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1908 00000100:00080000:0.0:1789761931.003828:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761931.003833:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761931.003838:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761931.003854:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507569152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761931.003869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761931.003897:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761931.003936:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761931.003964:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569152:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761931.067585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761932.091557:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761933.115500:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789761934.075720:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1789761934.075792:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789761934.075810:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761934.075863:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761934.075876:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761934.075887:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761934.075902:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789761934.075931:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789761934.075953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761934] req@ffff8d25d25b8000 x1876699507568640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761934.075993:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507568640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761934.076013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761934.076021:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761934.076028:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761934.076040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761934] req@ffff8d25d25ba300 x1876699507568768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761934.076062:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507568768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761934.076098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761934.076104:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761934.076110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761934.076121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761934] req@ffff8d25d25b8a80 x1876699507568896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761934.076141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507568896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761934.076154:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761934.076159:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761934.076163:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507568896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761934.076173:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761934] req@ffff8d25d25b9180 x1876699507569024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761934.076216:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507569024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761934.076228:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761934.076234:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761934.076239:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761934.076250:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761934] req@ffff8d25d25bbb80 x1876699507568512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761934.076271:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507568512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761934.076283:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761934.076288:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761934.076294:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507568512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761934.076339:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761931/real 1789761934] req@ffff8d25d25bad80 x1876699507569152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761934.076358:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507569152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761956 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761934.076370:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761934.076401:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761934.076408:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569152:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761934.139547:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761935.163637:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761936.123580:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761936.123590:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761936.123595:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761936.123605:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1913 00000100:00080000:0.0:1789761936.123616:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761936.123623:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761936.123671:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507569280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761936.123732:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761936.123737:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761936.123739:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761936.123758:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1913 00000100:00080000:0.0:1789761936.123761:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761936.123765:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761936.123775:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761936.123788:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507569408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761936.123795:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761936.123798:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761936.123799:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761936.123801:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1913 00000100:00080000:0.0:1789761936.123804:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761936.123807:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761936.123810:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761936.123817:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507569536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761936.123826:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761936.123830:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761936.123831:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761936.123835:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1913 00000100:00080000:0.0:1789761936.123839:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761936.123843:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761936.123848:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761936.123861:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507569664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761936.123871:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761936.123874:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761936.123876:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761936.123879:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1913 00000100:00080000:0.0:1789761936.123882:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761936.123887:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761936.123891:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761936.123902:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507569792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761936.123918:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761936.123922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761936.123924:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761936.123928:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1913 00000100:00080000:0.0:1789761936.123931:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761936.123934:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761936.123938:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761936.123946:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507569920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761936.124017:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507569280:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1789761936.124125:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761936.124173:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffff8d25d25ba680 x1876699507569280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761936.124195:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507569280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761936.124207:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761936.124212:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761936.124217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507569280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761936.124226:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507569408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761936.124252:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761936.124267:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffff8d25d25bb800 x1876699507569408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761936.124353:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507569408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761936.124363:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761936.124367:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761936.124370:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507569408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761936.124400:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507569536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761936.124425:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761936.124455:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffff8d25d25b9f80 x1876699507569536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761936.124468:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507569536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761936.124476:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761936.124480:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761936.124483:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507569536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761936.124490:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507569664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761936.124512:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761936.124525:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffff8d25d25b8380 x1876699507569664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761936.124537:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507569664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761936.124545:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761936.124548:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761936.124551:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507569664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761936.124557:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761936.124576:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761936.124587:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffff8d25d25b8700 x1876699507569792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761936.124598:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507569792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761936.124605:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761936.124608:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761936.124611:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761936.124617:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761936.124637:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761936.124648:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761936/real 1789761936] req@ffff8d25d25bb100 x1876699507569920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761936.124659:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507569920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761961 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761936.124675:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761936.124678:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761936.124681:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507569920:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761936.187597:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761937.211587:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761938.235581:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761939.259652:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761940.283575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761941.243458:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.243466:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761941.243470:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.243480:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1918 00000100:00080000:0.0:1789761941.243488:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.243494:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761941.243544:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507570048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761941.243571:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.243574:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761941.243576:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.243579:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1918 00000100:00080000:0.0:1789761941.243584:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.243588:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761941.243596:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761941.243608:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507570048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761941.243610:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507570176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761941.243622:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.243624:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761941.243626:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.243629:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1918 00000100:00080000:0.0:1789761941.243631:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.243634:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761941.243637:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761941.243645:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507570304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761941.243652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.243654:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761941.243655:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.243658:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1918 00000100:00080000:0.0:1789761941.243661:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.243663:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761941.243666:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761941.243670:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761941.243676:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507570432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761941.243690:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761941.243692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761941.243693:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.243696:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1918 00000100:00080000:0.0:1789761941.243698:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761941.243701:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761941.243705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761941.243708:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffff8d25d25b9c00 x1876699507570048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761941.243716:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507570560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761941.243725:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761941.243729:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507570048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761941.243730:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761941.243734:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761941.243742:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1918 00000100:00080000:0.0:1789761941.243749:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761941.243751:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761941.243752:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761941.243761:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789761941.243762:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761941.243768:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507570048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761941.243779:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507570688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761941.243795:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761941.243826:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761941.243844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffff8d25d25b8e00 x1876699507570176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761941.243856:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507570176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761941.243871:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761941.243875:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761941.243882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761941.243888:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761941.243903:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761941.243909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffff8d25d25bb480 x1876699507570304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761941.243917:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507570304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761941.243922:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761941.243924:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761941.243926:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761941.243930:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761941.243941:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761941.243947:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffff8d25d25bb100 x1876699507570432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761941.243954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507570432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761941.243959:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761941.243961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761941.243964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761941.243967:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507570560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761941.243977:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761941.243983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffff8d25d25b8700 x1876699507570560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761941.243990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507570560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761941.243995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761941.243997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761941.243998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507570560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761941.244001:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507570688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761941.244028:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761941.244038:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761941/real 1789761941] req@ffff8d25d25b8380 x1876699507570688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761941.244046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507570688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761966 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761941.244052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761941.244054:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761941.244056:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507570688:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761941.307586:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761942.331553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761943.355524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761944.379530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761945.403570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761946.363524:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.363535:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761946.363540:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.363552:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1924 00000100:00080000:0.0:1789761946.363566:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.363575:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761946.363616:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507570816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761946.363642:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.363645:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761946.363646:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.363649:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1924 00000100:00080000:0.0:1789761946.363652:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.363655:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761946.363666:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761946.363678:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507570944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761946.363690:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507570816:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789761946.363693:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.363704:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761946.363713:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.363718:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1924 00000100:00080000:0.0:1789761946.363724:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.363729:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761946.363734:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761946.363745:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507571072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761946.363751:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.363755:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761946.363757:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.363760:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1924 00000100:00080000:0.0:1789761946.363764:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.363769:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761946.363772:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761946.363781:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761946.363783:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507571200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761946.363791:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.363794:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761946.363796:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.363801:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1924 00000100:00080000:0.0:1789761946.363805:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.363810:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761946.363818:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761946.363834:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507571328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761946.363843:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761946.363847:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761946.363849:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761946.363853:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1924 00000100:00080000:0.0:1789761946.363857:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761946.363862:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761946.363870:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761946.363888:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507571456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761946.363895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffff8d25d25b9f80 x1876699507570816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761946.363953:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507570816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761946.363965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761946.363982:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761946.363988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507570816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761946.364023:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761946.364051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761946.364077:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffff8d25d25bb800 x1876699507570944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761946.364097:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507570944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761946.364105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761946.364107:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761946.364110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507570944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761946.364115:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761946.364131:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761946.364139:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffff8d25d25ba680 x1876699507571072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761946.364147:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507571072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761946.364152:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761946.364154:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761946.364156:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761946.364161:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761946.364181:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761946.364192:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffff8d25d25bad80 x1876699507571200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761946.364210:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507571200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761946.364216:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761946.364364:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761946.364367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761946.364372:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507571328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761946.364431:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761946.364447:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffff8d25d25bbb80 x1876699507571328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761946.364540:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507571328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761946.364547:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761946.364549:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761946.364551:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507571328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761946.364556:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507571456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761946.364574:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761946.364582:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761946/real 1789761946] req@ffff8d25d25b9180 x1876699507571456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761946.364590:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507571456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761971 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761946.364594:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761946.364596:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761946.364598:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507571456:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761946.427602:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761947.451573:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761948.475537:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761949.499554:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761950.523554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761951.483476:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761951.483488:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761951.483494:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761951.483504:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1929 00000100:00080000:2.0:1789761951.483514:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761951.483521:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761951.483564:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507571584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761951.483591:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761951.483593:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761951.483595:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761951.483599:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1929 00000100:00080000:2.0:1789761951.483602:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761951.483606:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761951.483614:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761951.483625:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507571712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761951.483634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507571584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761951.483635:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761951.483642:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761951.483645:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761951.483648:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1929 00000100:00080000:2.0:1789761951.483650:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761951.483653:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761951.483656:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761951.483664:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507571840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761951.483676:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761951.483678:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761951.483679:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761951.483682:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1929 00000100:00080000:2.0:1789761951.483685:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761951.483688:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761951.483691:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761951.483696:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761951.483701:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507571968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761951.483710:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761951.483714:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761951.483715:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761951.483719:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1929 00000100:00080000:2.0:1789761951.483723:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761951.483728:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761951.483734:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761951.483734:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761951/real 1789761951] req@ffff8d25c8020700 x1876699507571584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761951.483752:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507572096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761951.483754:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507571584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761951.483787:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761951.483792:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761951.483792:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761951.483794:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761951.483799:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761951.483802:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1929 00000100:00080000:2.0:1789761951.483806:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761951.483808:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507571584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761951.483812:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761951.483821:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761951.483830:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507572224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761951.483843:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761951.483868:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761951.483886:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761951/real 1789761951] req@ffff8d25c8023100 x1876699507571712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761951.483894:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507571712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761951.483900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761951.483902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761951.483904:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761951.483908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761951.483919:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761951.483925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761951/real 1789761951] req@ffff8d25c8023b80 x1876699507571840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761951.483932:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507571840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761951.483965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761951.483968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761951.483970:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761951.483975:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761951.483990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761951.483997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761951/real 1789761951] req@ffff8d25c8020380 x1876699507571968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761951.484015:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507571968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761951.484021:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761951.484023:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761951.484025:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507571968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761951.484028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761951.484037:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761951.484043:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761951/real 1789761951] req@ffff8d25c8022d80 x1876699507572096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761951.484050:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507572096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761951.484055:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761951.484057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761951.484059:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761951.484062:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761951.484071:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761951.484076:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761951/real 1789761951] req@ffff8d25c8022300 x1876699507572224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761951.484084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507572224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761976 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761951.484089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761951.484090:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761951.484092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572224:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761951.547509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761952.571553:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761953.595486:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761954.619535:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761955.643494:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761956.603509:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761956.603520:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761956.603525:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761956.603534:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1934 00000100:00080000:2.0:1789761956.603543:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761956.603551:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761956.603588:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507572352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761956.603619:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761956.603623:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761956.603625:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761956.603630:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1934 00000100:00080000:2.0:1789761956.603634:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761956.603640:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761956.603650:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761956.603665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507572352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789761956.603667:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507572480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761956.603681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761956.603685:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761956.603687:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761956.603691:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1934 00000100:00080000:2.0:1789761956.603695:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761956.603700:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761956.603706:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761956.603720:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507572608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761956.603727:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761956.603737:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761956.603741:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761956.603744:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761956.603749:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1934 00000100:00080000:2.0:1789761956.603753:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761956.603758:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761956.603764:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761956.603765:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761956/real 1789761956] req@ffff8d25c8021880 x1876699507572352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761956.603784:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507572736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761956.603785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507572352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761956.603820:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761956.603822:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761956.603825:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789761956.603831:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789761956.603832:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507572352:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761956.603835:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761956.603841:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1934 00000100:00080000:2.0:1789761956.603846:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761956.603851:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761956.603854:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507572480:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789761956.603860:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761956.603877:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761956.603880:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507572864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761956.603890:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761956/real 1789761956] req@ffff8d25c8021c00 x1876699507572480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761956.603915:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761956.603922:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761956.603924:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761956.603927:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507572480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761956.603932:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1934 00000100:00080000:2.0:1789761956.603940:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761956.603944:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789761956.603946:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789761956.603952:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789761956.603955:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761956.603959:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507572480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761956.603970:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507572608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789761956.603973:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507572992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761956.604000:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761956.604012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761956/real 1789761956] req@ffff8d25c8021500 x1876699507572608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761956.604026:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507572608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761956.604034:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761956.604038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761956.604041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507572608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761956.604048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507572736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761956.604070:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761956.604082:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761956/real 1789761956] req@ffff8d25c8020e00 x1876699507572736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761956.604095:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507572736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761956.604108:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761956.604111:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761956.604113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507572736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761956.604125:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761956.604140:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761956.604148:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761956/real 1789761956] req@ffff8d25c8023480 x1876699507572864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761956.604155:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507572864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761956.604160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761956.604162:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761956.604164:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761956.604168:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761956.604177:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761956.604183:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761956/real 1789761956] req@ffff8d25c8021880 x1876699507572992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761956.604190:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507572992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761981 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761956.604195:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761956.604197:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761956.604199:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507572992:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761956.667574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761957.691549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761958.715551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761959.739508:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761960.763564:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761961.723562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761961.723572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761961.723577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761961.723589:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1939 00000100:00080000:2.0:1789761961.723600:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761961.723608:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761961.723652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507573120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761961.723676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761961.723680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761961.723681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761961.723685:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1939 00000100:00080000:2.0:1789761961.723688:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761961.723691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761961.723701:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761961.723726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507573248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761961.723737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761961.723742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761961.723743:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507573120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761961.723744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761961.723754:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1939 00000100:00080000:2.0:1789761961.723757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761961.723760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761961.723764:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761961.723774:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507573376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761961.723782:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761961.723786:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761961.723787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761961.723790:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1939 00000100:00080000:2.0:1789761961.723793:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761961.723796:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761961.723800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789761961.723809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507573504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761961.723823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761961.723825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761961.723827:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761961.723830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1939 00000800:00000100:3.0:1789761961.723836:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789761961.723837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761961.723843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761961.723847:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761961.723856:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507573632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761961.723882:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761961.723885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761961.723887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761961.723891:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1939 00000100:00080000:2.0:1789761961.723913:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761961.723921:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761961.723932:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761961.723933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761961/real 1789761961] req@ffff8d25c8022300 x1876699507573120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761961.723960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507573760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761961.723966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507573120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761961.724016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761961.724024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761961.724030:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507573120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761961.724064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507573248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761961.724094:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761961.724109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761961/real 1789761961] req@ffff8d25c8022d80 x1876699507573248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761961.724123:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507573248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761961.724133:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761961.724137:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761961.724141:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507573248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761961.724148:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507573376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761961.724170:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761961.724189:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761961/real 1789761961] req@ffff8d25c8020380 x1876699507573376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761961.724209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507573376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761961.724224:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761961.724230:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761961.724236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507573376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761961.724247:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507573504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761961.724279:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761961.724293:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761961/real 1789761961] req@ffff8d25c8023b80 x1876699507573504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761961.724324:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507573504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761961.724333:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761961.724336:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761961.724339:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507573504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761961.724344:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507573632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761961.724422:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761961.724436:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761961/real 1789761961] req@ffff8d25c8023100 x1876699507573632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761961.724450:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507573632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761961.724459:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761961.724463:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761961.724466:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507573632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761961.724473:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507573760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761961.724491:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761961.724499:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761961/real 1789761961] req@ffff8d25c8020700 x1876699507573760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761961.724507:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507573760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761986 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761961.724513:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761961.724515:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761961.724517:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507573760:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761961.787565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761962.811533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761963.835516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761964.859539:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761965.883524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761966.843496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761966.843507:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761966.843511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761966.843522:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1944 00000100:00080000:1.0:1789761966.843531:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761966.843539:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761966.843576:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507573888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761966.843614:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761966.843621:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761966.843623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761966.843628:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1944 00000100:00080000:1.0:1789761966.843632:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761966.843636:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761966.843646:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761966.843659:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507574016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761966.843670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761966.843674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761966.843677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761966.843681:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1944 00000100:00080000:1.0:1789761966.843684:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761966.843688:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761966.843693:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761966.843705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507574144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761966.843717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761966.843721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761966.843723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761966.843732:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1944 00000100:00100000:3.0:1789761966.843734:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507573888:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761966.843737:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761966.843751:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761966.843756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761966.843768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507574272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761966.843777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761966.843779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761966.843781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761966.843784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1944 00000100:00080000:1.0:1789761966.843787:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761966.843790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761966.843794:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761966.843802:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507574400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761966.843809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761966.843812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761966.843813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761966.843816:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1944 00000100:00080000:1.0:1789761966.843830:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761966.843835:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761966.843839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761966.843841:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761966.843858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507574528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761966.843905:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761966/real 1789761966] req@ffff8d25c816a680 x1876699507573888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761966.843928:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507573888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761966.843937:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761966.843941:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761966.843944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507573888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761966.843951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761966.843970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761966.843979:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761966/real 1789761966] req@ffff8d25c816aa00 x1876699507574016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761966.843988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507574016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761966.843994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761966.843996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761966.843998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761966.844022:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761966.844034:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761966.844040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761966/real 1789761966] req@ffff8d25c8168e00 x1876699507574144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761966.844048:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507574144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761966.844058:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761966.844060:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761966.844062:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761966.844066:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761966.844077:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761966.844091:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761966/real 1789761966] req@ffff8d25c8169f80 x1876699507574272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761966.844099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507574272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761966.844105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761966.844107:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761966.844109:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761966.844113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507574400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761966.844132:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761966.844150:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761966/real 1789761966] req@ffff8d25c8168700 x1876699507574400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761966.844164:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507574400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761966.844173:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761966.844176:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761966.844180:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507574400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761966.844187:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507574528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761966.844204:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761966.844213:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761966/real 1789761966] req@ffff8d25c8169500 x1876699507574528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761966.844221:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507574528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761991 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761966.844227:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761966.844229:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761966.844231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507574528:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761966.907599:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761967.931546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761968.955596:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761969.979634:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761971.003573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789761971.963561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761971.963575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789761971.963581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761971.963593:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1949 00000100:00080000:2.0:1789761971.963609:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761971.963621:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789761971.963671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507574656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761971.963712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761971.963716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761971.963717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761971.963727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1949 00000100:00080000:2.0:1789761971.963730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761971.963734:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761971.963745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761971.963756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507574784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761971.963768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761971.963775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761971.963779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761971.963780:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507574656:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789761971.963843:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1949 00000100:00080000:2.0:1789761971.963849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761971.963858:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761971.963865:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761971.963881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507574912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761971.963899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761971.963902:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789761971.963904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761971.963909:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1949 00000100:00080000:2.0:1789761971.963912:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761971.963915:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761971.963919:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789761971.963922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789761971.963935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507575040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789761971.963945:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761971.963949:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789761971.963950:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761971.963953:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1949 00000100:00080000:2.0:1789761971.963956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761971.963959:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761971.963963:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789761971.963983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507575168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761971.963988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761971/real 1789761971] req@ffff8d25c8020000 x1876699507574656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789761971.963995:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789761971.964008:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789761971.964015:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789761971.964026:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1949 00000100:00080000:2.0:1789761971.964031:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789761971.964046:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789761971.964053:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761971.964058:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507574656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789761971.964066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507575296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761971.964119:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761971.964126:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761971.964131:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507574656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761971.964157:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761971.964193:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761971.964203:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761971/real 1789761971] req@ffff8d25c8023800 x1876699507574784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761971.964212:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507574784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761971.964218:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761971.964221:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761971.964223:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761971.964227:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761971.964238:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761971.964262:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761971/real 1789761971] req@ffff8d25c8022a00 x1876699507574912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761971.964270:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507574912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761971.964276:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761971.964278:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761971.964280:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507574912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761971.964284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761971.964297:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761971.964304:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761971/real 1789761971] req@ffff8d25c8022680 x1876699507575040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761971.964312:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507575040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761971.964318:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761971.964320:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761971.964322:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761971.964326:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507575168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761971.964358:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761971.964370:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761971/real 1789761971] req@ffff8d25c8021180 x1876699507575168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761971.964379:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507575168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761971.964387:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761971.964390:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761971.964394:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507575168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761971.964400:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507575296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761971.964429:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761971.964435:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761971/real 1789761971] req@ffff8d25c8020a80 x1876699507575296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761971.964443:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507575296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789761996 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761971.964452:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761971.964455:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761971.964457:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507575296:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761972.027543:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761973.051598:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761974.075583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761975.099517:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761976.123562:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789761977.083490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761977.083501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789761977.083505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761977.083514:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1954 00000100:00080000:3.0:1789761977.083524:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761977.083531:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789761977.083568:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507575424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761977.083597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761977.083601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761977.083603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761977.083607:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1954 00000100:00080000:3.0:1789761977.083611:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761977.083616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761977.083626:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761977.083641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507575552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761977.083644:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507575424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761977.083661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761977.083665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789761977.083667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761977.083672:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1954 00000100:00080000:3.0:1789761977.083677:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761977.083681:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761977.083687:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761977.083700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507575680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789761977.083705:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761977.083717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761977.083721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789761977.083723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761977.083728:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1954 00000100:00080000:3.0:1789761977.083733:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789761977.083737:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761977.083756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761977.083769:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffff8d25d1b08a80 x1876699507575424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761977.083772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507575808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761977.083798:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507575424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761977.083801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761977.083810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789761977.083814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761977.083819:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761977.083821:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1954 00000100:00080000:0.0:1789761977.083825:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789761977.083828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761977.083830:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507575424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789761977.083836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761977.083842:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761977.083855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761977.083858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507575936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761977.083873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761977.083877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:0.0:1789761977.083879:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789761977.083882:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761977.083889:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1954 00000100:00080000:3.0:1789761977.083894:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789761977.083895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffff8d25d1b0aa00 x1876699507575552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761977.083901:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789761977.083911:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761977.083915:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507575552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761977.083923:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761977.083928:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761977.083929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507576064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789761977.083932:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761977.083941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761977.083961:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761977.083970:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffff8d25d1b09180 x1876699507575680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761977.083982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507575680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761977.083991:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761977.083995:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761977.083998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761977.084005:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789761977.084027:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761977.084039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffff8d25d1b0ad80 x1876699507575808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761977.084052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507575808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761977.084063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761977.084066:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761977.084068:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507575808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789761977.084084:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507575936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761977.084099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761977.084109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffff8d25d1b09c00 x1876699507575936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761977.084121:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507575936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761977.084129:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761977.084132:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761977.084135:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507575936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789761977.084141:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507576064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789761977.084160:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789761977.084171:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761977/real 1789761977] req@ffff8d25d1b0b100 x1876699507576064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789761977.084183:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507576064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762002 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789761977.084191:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789761977.084194:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789761977.084198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507576064:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761977.147502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761978.171523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761979.195518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761980.219550:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761981.243496:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789761982.203475:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.203484:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789761982.203488:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.203501:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1959 00000100:00080000:0.0:1789761982.203513:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761982.203523:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789761982.203570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507576192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761982.203598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.203602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761982.203603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.203607:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1959 00000100:00080000:0.0:1789761982.203614:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761982.203619:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761982.203630:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761982.203642:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507576192:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789761982.203651:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507576320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761982.203661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.203664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761982.203666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.203677:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1959 00000100:00080000:0.0:1789761982.203682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761982.203685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761982.203689:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761982.203702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507576448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761982.203722:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.203725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789761982.203726:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.203730:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1959 00000100:00080000:0.0:1789761982.203734:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789761982.203734:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789761982.203745:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761982.203750:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789761982.203762:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507576576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761982.203769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.203771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789761982.203773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.203775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1959 00000100:00080000:0.0:1789761982.203777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761982.203780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761982.203784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761982.203796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507576704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789761982.203809:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789761982.203812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789761982.203815:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789761982.203819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1959 00000100:00080000:0.0:1789761982.203822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789761982.203825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789761982.203829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789761982.203837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507576832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761982.203862:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffff8d25d25b8a80 x1876699507576192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761982.203879:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507576192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761982.204004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761982.204008:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761982.204012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507576192:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761982.204038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507576320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761982.204111:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761982.204124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffff8d25d25ba300 x1876699507576320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761982.204133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507576320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761982.204139:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761982.204142:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761982.204144:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507576320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761982.204149:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507576448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761982.204162:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761982.204169:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffff8d25d25b8000 x1876699507576448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761982.204177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507576448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761982.204182:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761982.204185:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761982.204187:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507576448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761982.204190:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507576576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761982.204201:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761982.204207:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffff8d25d25b9180 x1876699507576576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761982.204215:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507576576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761982.204220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761982.204222:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761982.204224:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507576576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761982.204227:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507576704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761982.204238:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761982.204244:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffff8d25d25bbb80 x1876699507576704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761982.204252:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507576704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761982.204257:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761982.204259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761982.204260:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507576704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761982.204264:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507576832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761982.204274:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761982.204280:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761982/real 1789761982] req@ffff8d25d25bad80 x1876699507576832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761982.204292:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507576832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762007 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761982.204297:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761982.204299:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761982.204301:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507576832:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761982.267568:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761983.291545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761984.315583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761985.339531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761986.363548:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761987.323502:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761987.323512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761987.323517:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761987.323527:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1965 00000100:00080000:1.0:1789761987.323537:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761987.323546:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761987.323591:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507576960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761987.323624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761987.323628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761987.323630:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761987.323636:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1965 00000100:00080000:1.0:1789761987.323640:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761987.323645:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761987.323654:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761987.323659:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507576960:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761987.323683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507577088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761987.323696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761987.323700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761987.323702:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761987.323706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1965 00000100:00080000:1.0:1789761987.323710:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761987.323714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761987.323720:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789761987.323728:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789761987.323734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507577216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761987.323745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761987.323748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761987.323750:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761987.323754:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1965 00000100:00080000:1.0:1789761987.323759:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761987.323763:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761987.323768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789761987.323769:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffff8d25c8168000 x1876699507576960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789761987.323780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507577344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761987.323787:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507576960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761987.323796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789761987.323802:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789761987.323803:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761987.323807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789761987.323810:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789761987.323814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1965 00000100:00100000:3.0:1789761987.323816:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507576960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761987.323819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761987.323825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761987.323829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761987.323838:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761987.323843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507577472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761987.323855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761987.323860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:3.0:1789761987.323861:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789761987.323863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761987.323869:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1965 00000100:00080000:1.0:1789761987.323874:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761987.323875:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffff8d25c8168380 x1876699507577088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789761987.323880:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761987.323888:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761987.323893:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507577088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761987.323902:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789761987.323903:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507577600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789761987.323909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761987.323917:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761987.323927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761987.323949:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761987.323959:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffff8d25c816b480 x1876699507577216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761987.323967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507577216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761987.323972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761987.323974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761987.323976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761987.323980:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761987.323990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761987.323996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffff8d25c816a300 x1876699507577344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761987.324003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507577344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761987.324008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761987.324010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761987.324012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761987.324023:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507577472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761987.324033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761987.324040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffff8d25c816ad80 x1876699507577472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761987.324047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507577472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761987.324052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761987.324053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761987.324055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507577472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761987.324059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507577600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761987.324068:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761987.324074:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761987/real 1789761987] req@ffff8d25c8168a80 x1876699507577600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761987.324081:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507577600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762012 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761987.324086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761987.324087:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761987.324089:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507577600:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789761987.387501:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761988.411525:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761989.435495:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761990.459570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761991.483579:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761992.443460:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761992.443468:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761992.443471:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761992.443482:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1970 00000100:00080000:1.0:1789761992.443490:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761992.443496:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761992.443528:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507577728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761992.443551:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761992.443554:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761992.443556:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761992.443559:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1970 00000100:00080000:1.0:1789761992.443561:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761992.443563:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761992.443570:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761992.443578:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507577856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761992.443595:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761992.443597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761992.443598:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761992.443601:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1970 00000100:00080000:1.0:1789761992.443603:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761992.443605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761992.443608:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761992.443615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507577984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761992.443622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761992.443626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761992.443640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761992.443645:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1970 00000100:00080000:1.0:1789761992.443649:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761992.443652:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761992.443668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761992.443677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507578112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761992.443685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761992.443690:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761992.443693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789761992.443693:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507577728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789761992.443708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1970 00000100:00080000:1.0:1789761992.443713:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761992.443717:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761992.443720:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761992.443730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507578240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761992.443736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761992.443739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761992.443741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761992.443745:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1970 00000100:00080000:1.0:1789761992.443747:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761992.443750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761992.443753:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761992.443760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507578368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789761992.443789:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761992.443836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761992/real 1789761992] req@ffff8d25c8169880 x1876699507577728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761992.443852:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507577728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761992.443863:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761992.443866:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761992.443869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507577728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789761992.443875:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761992.443890:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761992.443898:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761992/real 1789761992] req@ffff8d25c8169c00 x1876699507577856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761992.443907:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507577856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761992.443913:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761992.443915:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761992.443917:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761992.443923:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761992.443942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761992.443954:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761992/real 1789761992] req@ffff8d25c8168a80 x1876699507577984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761992.443975:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507577984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761992.443984:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761992.443987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761992.443990:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507577984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761992.444016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789761992.444033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761992.444041:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761992/real 1789761992] req@ffff8d25c816ad80 x1876699507578112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761992.444049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507578112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761992.444054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761992.444056:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761992.444059:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789761992.444063:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507578240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761992.444073:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761992.444080:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761992/real 1789761992] req@ffff8d25c816a300 x1876699507578240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761992.444087:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507578240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761992.444096:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761992.444099:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761992.444101:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507578240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789761992.444104:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507578368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789761992.444114:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789761992.444120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761992/real 1789761992] req@ffff8d25c816b480 x1876699507578368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789761992.444127:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507578368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762017 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789761992.444133:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789761992.444135:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789761992.444139:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507578368:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789761992.507544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761993.531538:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789761994.555544:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789761995.579589:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761996.603553:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789761997.563484:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761997.563492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789761997.563496:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761997.563504:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1975 00000100:00080000:1.0:1789761997.563513:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761997.563519:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789761997.563558:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507578496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761997.563588:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761997.563608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761997.563610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761997.563616:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1975 00000100:00080000:1.0:1789761997.563627:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761997.563631:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761997.563640:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761997.563652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507578496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789761997.563655:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507578624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761997.563670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761997.563673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789761997.563674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761997.563682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1975 00000100:00080000:1.0:1789761997.563685:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761997.563688:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761997.563691:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761997.563700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507578752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761997.563706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761997.563709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789761997.563710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761997.563713:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1975 00000100:00080000:1.0:1789761997.563716:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761997.563718:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761997.563721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789761997.563730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507578880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761997.563737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789761997.563739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789761997.563741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761997.563744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1975 00000100:00080000:1.0:1789761997.563748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789761997.563753:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761997.563754:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761997.563762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789761997.563773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507579008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789761997.563783:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789761997.563789:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578752:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789761997.563806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789761997.563807:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578880:192.168.203.141@tcp:8:kworker.0 00000100:00080000:1.0:1789761997.563812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789761997.563817:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1975 00000100:00080000:1.0:1789761997.563821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789761997.563824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789761997.563829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789761997.563833:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789761997.563845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507579136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789761997.563859:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579136:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789761997.627541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761998.651542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789761999.675568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762000.635705:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1789762000.635745:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762000.635759:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762000.635797:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762000.635806:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762000.635817:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789762000.635819:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761997/real 1789762000] req@ffff8d25c8168000 x1876699507578624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789762000.635826:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762000.635836:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789762000.635850:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507578624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762000.635860:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762000.635864:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762000.635867:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762000.635873:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761997/real 1789762000] req@ffff8d25c8169500 x1876699507578752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762000.635886:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507578752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762000.635892:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762000.635894:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762000.635896:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762000.635900:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761997/real 1789762000] req@ffff8d25c8168700 x1876699507578880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762000.635907:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507578880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762000.635912:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762000.635915:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762000.635917:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507578880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762000.635921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761997/real 1789762000] req@ffff8d25c8168380 x1876699507578496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762000.635929:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507578496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762000.635933:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762000.635935:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762000.635937:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507578496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762000.635955:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761997/real 1789762000] req@ffff8d25c8168e00 x1876699507579136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762000.635968:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507579136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762000.635974:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762000.635977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762000.635979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762000.635982:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789761997/real 1789762000] req@ffff8d25c8169f80 x1876699507579008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762000.635988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507579008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762022 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762000.635993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762000.635995:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762000.635996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579008:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762000.699486:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762001.723621:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762002.683589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762002.683600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762002.683604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762002.683614:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1980 00000100:00080000:0.0:1789762002.683625:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762002.683632:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762002.683664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507579264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762002.683694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762002.683699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762002.683701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762002.683705:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1980 00000100:00080000:0.0:1789762002.683708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762002.683713:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762002.683724:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762002.683737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507579392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762002.683748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762002.683752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762002.683754:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762002.683764:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1980 00000100:00100000:3.0:1789762002.683764:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507579264:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762002.683770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762002.683779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762002.683785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762002.683799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507579520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762002.683808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762002.683812:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762002.683814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762002.683818:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1980 00000100:00080000:0.0:1789762002.683823:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762002.683827:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762002.683833:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762002.683845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507579648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762002.683859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762002.683862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762002.683864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762002.683869:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1980 00000100:00080000:0.0:1789762002.683875:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762002.683881:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762002.683889:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762002.683896:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762002.683913:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507579776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762002.683924:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762002.683929:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762002.683931:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762002.683938:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1980 00000100:00080000:0.0:1789762002.683945:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762002.683950:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762002.683957:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762002.683961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762002/real 1789762002] req@ffff8d25d25ba680 x1876699507579264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762002.683975:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507579904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762002.683996:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507579264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762002.684013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762002.684021:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762002.684026:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507579264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762002.684064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507579392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762002.684094:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762002.684109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762002/real 1789762002] req@ffff8d25d25bb800 x1876699507579392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762002.684129:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507579392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762002.684140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762002.684144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762002.684148:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507579392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762002.684156:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507579520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762002.684189:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762002.684204:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762002/real 1789762002] req@ffff8d25d25b9f80 x1876699507579520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762002.684216:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507579520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762002.684353:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762002.684358:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762002.684362:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507579520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762002.684370:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507579648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762002.684435:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762002.684449:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762002/real 1789762002] req@ffff8d25d25b8380 x1876699507579648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762002.684461:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507579648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762002.684475:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762002.684479:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762002.684482:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507579648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762002.684489:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762002.684513:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762002.684524:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762002/real 1789762002] req@ffff8d25d25b8700 x1876699507579776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762002.684534:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507579776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762002.684541:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762002.684544:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762002.684547:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762002.684570:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762002.684593:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762002.684605:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762002/real 1789762002] req@ffff8d25d25bb100 x1876699507579904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762002.684617:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507579904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762027 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762002.684625:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762002.684628:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762002.684632:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507579904:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762002.747480:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762003.771509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762004.795511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762005.819574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762006.843502:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762007.803480:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762007.803490:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762007.803495:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762007.803505:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1985 00000100:00080000:0.0:1789762007.803527:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762007.803535:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762007.803569:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507580032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762007.803595:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762007.803600:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762007.803602:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762007.803606:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1985 00000100:00080000:0.0:1789762007.803610:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762007.803615:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762007.803624:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762007.803637:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507580032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762007.803640:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507580160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762007.803652:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762007.803655:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762007.803656:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762007.803659:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1985 00000100:00080000:0.0:1789762007.803662:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762007.803664:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762007.803668:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762007.803678:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507580288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762007.803689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762007.803692:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762007.803693:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762007.803697:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1985 00000100:00080000:0.0:1789762007.803700:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762007.803702:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762007.803706:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762007.803711:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762007.803718:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507580416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762007.803728:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762007.803731:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762007.803732:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762007.803734:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1985 00000100:00080000:0.0:1789762007.803736:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762007.803739:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762007.803742:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762007.803751:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762007/real 1789762007] req@ffff8d25d25bb480 x1876699507580032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762007.803753:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507580544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762007.803780:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762007.803782:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762007.803783:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762007.803786:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1985 00000100:00080000:0.0:1789762007.803789:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762007.803792:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762007.803795:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762007.803797:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507580032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762007.803809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762007.803812:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507580672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762007.803815:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762007.803822:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507580032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762007.803847:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762007.803866:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762007.803875:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762007/real 1789762007] req@ffff8d25d25b8e00 x1876699507580160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762007.803884:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507580160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762007.803890:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762007.803892:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762007.803895:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762007.803901:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762007.803912:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762007.803919:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762007/real 1789762007] req@ffff8d25d25b9c00 x1876699507580288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762007.803927:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507580288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762007.803932:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762007.803934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762007.803936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762007.803939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762007.803958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762007.803964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762007/real 1789762007] req@ffff8d25d25bb100 x1876699507580416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762007.803972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507580416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762007.803978:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762007.803980:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762007.803982:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762007.803985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507580544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762007.803995:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762007.804003:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762007/real 1789762007] req@ffff8d25d25b8700 x1876699507580544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762007.804016:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507580544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762007.804024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762007.804028:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762007.804031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507580544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762007.804038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507580672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762007.804058:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762007.804067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762007/real 1789762007] req@ffff8d25d25b8380 x1876699507580672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762007.804075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507580672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762032 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762007.804080:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762007.804082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762007.804084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507580672:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762007.867502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762008.891583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762009.915589:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762010.939556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762011.963538:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762012.923503:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762012.923514:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762012.923518:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762012.923528:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1990 00000100:00080000:1.0:1789762012.923536:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762012.923542:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762012.923577:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507580800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762012.923604:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762012.923609:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762012.923612:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762012.923618:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1990 00000100:00080000:1.0:1789762012.923622:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762012.923633:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507580800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762012.923637:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762012.923650:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762012.923667:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507580928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762012.923681:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762012.923684:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762012.923687:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762012.923691:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762012.923695:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1990 00000100:00080000:1.0:1789762012.923702:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762012.923707:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762012.923713:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762012.923726:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762012/real 1789762012] req@ffff8d25c816aa00 x1876699507580800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762012.923730:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507581056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762012.923763:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762012.923768:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507580800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762012.923770:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762012.923775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762012.923781:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762012.923783:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1990 00000100:00080000:3.0:1789762012.923787:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762012.923793:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762012.923793:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507580800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762012.923804:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762012.923811:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762012.923818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762012.923827:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507581184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762012.923839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762012.923846:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762012.923852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762012/real 1789762012] req@ffff8d25c816a680 x1876699507580928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762012.923853:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762012.923858:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762012.923868:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1990 00000100:00080000:1.0:1789762012.923874:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762012.923876:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507580928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762012.923881:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762012.923892:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762012.923895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762012.923901:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762012.923910:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507581312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762012.923910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507580928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762012.923923:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762012.923934:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762012.923938:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762012.923941:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762012.923947:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762012.923948:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1990 00000100:00080000:1.0:1789762012.923957:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762012.923960:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762012/real 1789762012] req@ffff8d25c8169f80 x1876699507581056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762012.923965:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762012.923973:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507581056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762012.923975:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762012.923989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762012.923993:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762012.923995:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507581440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762012.923996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762012.924011:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762012.924026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762012.924033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762012/real 1789762012] req@ffff8d25c8168e00 x1876699507581184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762012.924041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507581184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762012.924047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762012.924049:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762012.924051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762012.924054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507581312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762012.924065:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762012.924071:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762012/real 1789762012] req@ffff8d25c8168380 x1876699507581312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762012.924078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507581312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762012.924083:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762012.924085:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762012.924087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507581312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762012.924094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507581440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762012.924104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762012.924110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762012/real 1789762012] req@ffff8d25c8168700 x1876699507581440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762012.924117:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507581440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762037 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762012.924122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762012.924123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762012.924125:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507581440:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762012.987511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762014.011558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762015.035556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762016.059574:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762017.083573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762018.043484:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762018.043495:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762018.043500:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762018.043510:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 1995 00000100:00080000:1.0:1789762018.043521:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762018.043528:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762018.043560:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507581568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762018.043582:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762018.043584:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762018.043586:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762018.043589:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1995 00000100:00080000:1.0:1789762018.043591:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762018.043594:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762018.043602:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762018.043610:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507581696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762018.043616:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762018.043619:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762018.043620:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762018.043622:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1995 00000100:00080000:1.0:1789762018.043624:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762018.043626:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762018.043628:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762018.043633:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507581824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762018.043644:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762018.043646:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762018.043647:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762018.043650:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 1995 00000100:00080000:1.0:1789762018.043651:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762018.043653:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762018.043656:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762018.043661:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507581952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762018.043667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762018.043669:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762018.043670:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762018.043671:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1995 00000100:00080000:1.0:1789762018.043673:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762018.043675:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762018.043678:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762018.043683:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507582080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762018.043689:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762018.043691:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762018.043691:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762018.043693:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 1995 00000100:00080000:1.0:1789762018.043696:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762018.043698:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762018.043701:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762018.043706:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507582208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762018.043976:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507581568:192.168.203.141@tcp:250:kworker.0 00000800:00000100:3.0:1789762018.044116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762018.044154:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762018] req@ffff8d25c8169500 x1876699507581568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762018.044173:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507581568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762018.044180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762018.044183:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762018.044186:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507581568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762018.044193:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762018.044211:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762018.044220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762018] req@ffff8d25c8168000 x1876699507581696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762018.044325:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507581696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762018.044331:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762018.044334:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762018.044338:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762018.044342:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762018.044361:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762018.044492:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762018] req@ffff8d25c816b480 x1876699507581824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762018.044511:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507581824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762018.044522:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762018.044527:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762018.044531:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762018.044554:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762018.044583:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762018.044598:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762018] req@ffff8d25c816a300 x1876699507581952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762018.044612:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507581952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762018.044621:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762018.044624:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762018.044628:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507581952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762018.044634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762018.044656:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762018.044668:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762018] req@ffff8d25c816ad80 x1876699507582080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762018.044680:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507582080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762018.044689:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762018.044692:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762018.044696:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762018.044701:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762018.044721:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762018.044732:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762018/real 1789762018] req@ffff8d25c8168a80 x1876699507582208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762018.044744:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507582208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762043 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762018.044759:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762018.044762:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762018.044765:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582208:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762018.107545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762019.131525:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762020.155529:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762021.179541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762022.203514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762023.163476:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762023.163486:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762023.163501:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762023.163516:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2000 00000100:00080000:1.0:1789762023.163525:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762023.163534:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762023.163570:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507582336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762023.163599:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762023.163603:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762023.163606:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762023.163611:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2000 00000100:00080000:1.0:1789762023.163616:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762023.163622:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762023.163632:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507582336:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789762023.163633:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762023.163653:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507582464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762023.163667:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762023.163671:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762023.163673:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762023.163677:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2000 00000100:00080000:1.0:1789762023.163683:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762023.163688:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762023.163693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789762023.163698:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762023.163727:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffff8d25c8169c00 x1876699507582336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762023.163741:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507582592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762023.163744:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507582336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762023.163766:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762023.163769:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762023.163774:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507582336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762023.163783:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762023.163789:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762023.163791:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762023.163796:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2000 00000100:00080000:1.0:1789762023.163802:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762023.163806:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507582464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762023.163810:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762023.163820:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762023.163836:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507582720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762023.163848:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762023.163851:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762023.163856:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762023.163859:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762023.163864:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffff8d25c8169880 x1876699507582464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762023.163867:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2000 00000100:00080000:1.0:1789762023.163875:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762023.163880:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762023.163892:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762023.163901:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507582464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762023.163909:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507582848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762023.163942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762023.163945:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762023.163949:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762023.163955:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762023.163957:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507582464:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762023.163959:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762023.163965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507582592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762023.163968:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2000 00000100:00080000:1.0:1789762023.163974:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762023.163979:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762023.163986:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762023.163990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762023.164005:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507582976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762023.164005:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffff8d25c8168a80 x1876699507582592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762023.164028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507582592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762023.164036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762023.164039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762023.164041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507582592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762023.164062:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507582720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762023.164076:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762023.164084:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffff8d25c816ad80 x1876699507582720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762023.164091:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507582720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762023.164097:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762023.164099:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762023.164101:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507582720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762023.164105:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762023.164116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762023.164122:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffff8d25c816a300 x1876699507582848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762023.164129:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507582848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762023.164135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762023.164136:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762023.164138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762023.164141:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762023.164151:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762023.164157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762023/real 1789762023] req@ffff8d25c816b480 x1876699507582976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762023.164165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507582976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762048 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762023.164170:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762023.164171:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762023.164173:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507582976:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762023.227571:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762024.251515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762025.275592:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762026.299533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762027.323573:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762028.283473:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762028.283484:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762028.283489:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762028.283499:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2005 00000100:00080000:0.0:1789762028.283509:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762028.283516:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762028.283559:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507583104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762028.283585:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762028.283588:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762028.283589:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762028.283593:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2005 00000100:00080000:0.0:1789762028.283596:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762028.283599:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762028.283609:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762028.283617:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507583104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762028.283621:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507583232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762028.283631:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762028.283634:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762028.283636:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762028.283639:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2005 00000100:00080000:0.0:1789762028.283641:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762028.283643:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762028.283646:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762028.283654:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507583360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762028.283660:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762028.283663:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762028.283664:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762028.283667:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2005 00000100:00080000:0.0:1789762028.283669:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762028.283671:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762028.283674:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762028.283681:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507583488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762028.283688:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762028.283690:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762028.283692:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762028.283694:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2005 00000100:00080000:0.0:1789762028.283696:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762028.283701:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762028.283702:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762028.283707:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762028.283715:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507583616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762028.283722:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762028.283724:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762028.283725:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762028.283728:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2005 00000100:00080000:0.0:1789762028.283730:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762028.283733:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762028.283736:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762028.283744:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507583744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762028.283752:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffff8d25d25b9f80 x1876699507583104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.283769:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507583104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.283779:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.283782:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762028.283785:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507583104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762028.283812:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507583232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762028.283838:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762028.283852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffff8d25d25bb800 x1876699507583232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.283867:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507583232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.283874:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.283876:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762028.283879:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507583232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762028.283883:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507583360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762028.283900:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762028.283908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffff8d25d25ba680 x1876699507583360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.283925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507583360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.283931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.283933:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762028.283935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507583360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762028.283939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507583488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762028.283952:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762028.283959:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffff8d25d25bad80 x1876699507583488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.283967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507583488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.283973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.283974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762028.283976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507583488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762028.283979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507583616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762028.283991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762028.283997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffff8d25d25bbb80 x1876699507583616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.284004:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507583616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.284015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.284017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762028.284019:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507583616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762028.284023:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507583744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762028.284033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762028.284039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762028/real 1789762028] req@ffff8d25d25b9180 x1876699507583744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762028.284046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507583744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762053 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762028.284052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762028.284053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762028.284055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507583744:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762028.347557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762029.371577:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762030.395562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762031.419533:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762032.443665:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762033.403475:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762033.403485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762033.403490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762033.403500:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2011 00000100:00080000:0.0:1789762033.403510:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762033.403518:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762033.403552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507583872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762033.403576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762033.403580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762033.403582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762033.403586:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2011 00000100:00080000:0.0:1789762033.403591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762033.403596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762033.403605:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762033.403621:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507583872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762033.403623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507584000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762033.403637:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762033.403640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762033.403643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762033.403647:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2011 00000100:00080000:0.0:1789762033.403651:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762033.403655:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762033.403661:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762033.403674:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507584128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762033.403679:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762033.403690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762033.403694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762033.403696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762033.403701:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2011 00000100:00080000:0.0:1789762033.403706:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762033.403711:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762033.403715:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762033/real 1789762033] req@ffff8d25d25b8000 x1876699507583872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762033.403719:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762033.403734:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507583872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762033.403738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507584256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762033.403770:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762033.403775:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762033.403780:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507583872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762033.403796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762033.403800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762033.403802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762033.403806:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584000:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762033.403810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2011 00000100:00080000:0.0:1789762033.403815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762033.403819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762033.403825:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762033.403829:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762033.403841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507584384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762033.403843:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762033/real 1789762033] req@ffff8d25d25ba300 x1876699507584000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762033.403873:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762033.403877:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507584000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762033.403880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762033.403884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762033.403891:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762033.403894:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2011 00000100:00080000:3.0:1789762033.403897:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762033.403900:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762033.403901:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584000:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762033.403908:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762033.403911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584128:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789762033.403915:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762033.403932:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762033.403933:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507584512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762033.403942:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762033/real 1789762033] req@ffff8d25d25b8a80 x1876699507584128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762033.403952:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507584128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762033.403958:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762033.403961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762033.403963:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762033.403968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762033.403979:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762033.403985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762033/real 1789762033] req@ffff8d25d25b9180 x1876699507584256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762033.403992:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507584256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762033.403998:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762033.404000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762033.404001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762033.404011:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507584384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762033.404022:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762033.404028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762033/real 1789762033] req@ffff8d25d25bbb80 x1876699507584384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762033.404035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507584384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762033.404040:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762033.404043:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762033.404045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507584384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762033.404048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507584512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762033.404058:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762033.404063:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762033/real 1789762033] req@ffff8d25d25bad80 x1876699507584512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762033.404071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507584512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762058 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762033.404076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762033.404086:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762033.404088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507584512:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762033.467519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762034.491647:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762035.515500:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762036.539584:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762037.563518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762038.523470:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762038.523478:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762038.523483:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762038.523492:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2016 00000100:00080000:0.0:1789762038.523500:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762038.523517:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762038.523548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507584640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762038.523582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762038.523585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762038.523586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762038.523589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2016 00000100:00080000:0.0:1789762038.523591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762038.523594:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762038.523601:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762038.523610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507584768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762038.523618:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762038.523623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762038.523624:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507584640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762038.523625:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762038.523633:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2016 00000100:00080000:0.0:1789762038.523646:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762038.523648:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762038.523651:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762038.523661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507584896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762038.523667:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762038.523670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762038.523672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762038.523674:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2016 00000100:00080000:0.0:1789762038.523677:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762038.523679:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762038.523682:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762038.523689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507585024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762038.523696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762038.523700:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762038.523713:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762038.523715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762038.523719:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2016 00000100:00080000:0.0:1789762038.523723:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762038.523727:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762038.523733:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762038.523746:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762038/real 1789762038] req@ffff8d25d25ba680 x1876699507584640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762038.523750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507585152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762038.523769:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762038.523770:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507584640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762038.523781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762038.523785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762038.523787:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762038.523791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2016 00000100:00080000:3.0:1789762038.523794:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762038.523798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762038.523800:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507584640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762038.523805:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762038.523811:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762038.523820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507585280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762038.523823:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762038.523855:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762038.523864:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762038/real 1789762038] req@ffff8d25d25bb800 x1876699507584768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762038.523887:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507584768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762038.523893:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762038.523895:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762038.523897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762038.523901:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762038.523913:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762038.523920:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762038/real 1789762038] req@ffff8d25d25b9f80 x1876699507584896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762038.523927:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507584896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762038.523932:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762038.523934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762038.523935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507584896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762038.523939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762038.523949:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762038.523955:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762038/real 1789762038] req@ffff8d25d25b8380 x1876699507585024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762038.523962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507585024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762038.523971:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762038.523974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762038.523976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762038.523979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507585152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762038.523988:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762038.523994:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762038/real 1789762038] req@ffff8d25d25b8700 x1876699507585152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762038.524001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507585152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762038.524006:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762038.524008:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762038.524010:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507585152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762038.524020:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507585280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762038.524031:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762038.524037:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762038/real 1789762038] req@ffff8d25d25bb100 x1876699507585280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762038.524045:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507585280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762063 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762038.524049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762038.524051:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762038.524053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507585280:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762038.587552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762039.611511:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762040.635569:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762041.659569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762042.683561:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762043.643470:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762043.643478:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762043.643481:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762043.643489:0:37:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2021 00000100:00080000:0.0:1789762043.643498:0:37:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762043.643504:0:37:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762043.643537:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507585408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762043.643559:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762043.643562:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762043.643563:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762043.643572:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2021 00000100:00080000:0.0:1789762043.643575:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762043.643578:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762043.643586:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762043.643596:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507585536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762043.643603:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507585408:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762043.643607:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762043.643611:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762043.643612:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762043.643615:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2021 00000100:00080000:0.0:1789762043.643618:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762043.643621:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762043.643624:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762043.643631:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507585664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762043.643638:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762043.643640:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762043.643642:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762043.643644:0:37:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2021 00000100:00080000:0.0:1789762043.643647:0:37:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762043.643649:0:37:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762043.643653:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762043.643663:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762043.643665:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507585792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762043.643680:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762043.643683:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762043.643685:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762043.643689:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2021 00000100:00080000:0.0:1789762043.643693:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762043.643698:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762043.643705:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762043.643706:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762043/real 1789762043] req@ffff8d25d25b9c00 x1876699507585408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762043.643723:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507585920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762043.643724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507585408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762043.643756:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762043.643760:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762043.643763:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507585408:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762043.643771:0:37:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762043.643774:0:37:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762043.643775:0:37:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762043.643779:0:37:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2021 00000100:00080000:0.0:1789762043.643783:0:37:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762043.643786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585536:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762043.643790:0:37:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762043.643797:0:37:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762043.643809:0:37:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507586048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762043.643814:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762043.643831:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762043/real 1789762043] req@ffff8d25d25b8e00 x1876699507585536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762043.643843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507585536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762043.643849:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762043.643851:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762043.643854:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762043.643861:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762043.643877:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762043.643887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762043/real 1789762043] req@ffff8d25d25bb480 x1876699507585664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762043.643896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507585664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762043.643902:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762043.643905:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762043.643908:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762043.643912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762043.643922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762043.643928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762043/real 1789762043] req@ffff8d25d25bb100 x1876699507585792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762043.643935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507585792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762043.643940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762043.643942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762043.643943:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507585792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762043.643946:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507585920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762043.643956:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762043.643962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762043/real 1789762043] req@ffff8d25d25b8700 x1876699507585920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762043.643970:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507585920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762043.643975:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762043.643977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762043.643978:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507585920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762043.643989:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507586048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762043.643999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762043.644006:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762043/real 1789762043] req@ffff8d25d25b8380 x1876699507586048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762043.644013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507586048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762068 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762043.644018:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762043.644020:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762043.644022:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507586048:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762043.707499:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762044.731514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762045.755485:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762046.779569:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762047.803542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762048.763467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762048.763477:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762048.763481:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762048.763492:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2026 00000100:00080000:0.0:1789762048.763501:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762048.763508:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762048.763550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507586176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762048.763582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762048.763586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762048.763588:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762048.763592:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2026 00000100:00080000:0.0:1789762048.763596:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762048.763599:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762048.763609:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762048.763623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507586304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762048.763630:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507586176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762048.763642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762048.763646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762048.763648:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762048.763653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2026 00000100:00080000:0.0:1789762048.763656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762048.763661:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762048.763666:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762048.763679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507586432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762048.763689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762048.763692:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762048.763696:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762048.763699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762048.763705:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2026 00000100:00080000:0.0:1789762048.763710:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762048.763714:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762048.763719:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762048.763734:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762048/real 1789762048] req@ffff8d25d25b9f80 x1876699507586176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762048.763735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507586560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762048.763766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762048.763770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762048.763772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762048.763777:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2026 00000100:00080000:0.0:1789762048.763780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762048.763784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762048.763789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762048.763790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507586176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762048.763803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507586688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762048.763807:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762048.763818:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762048.763821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762048.763827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762048.763827:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507586176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762048.763830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762048.763837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2026 00000100:00080000:0.0:1789762048.763841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762048.763846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762048.763851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762048.763855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507586304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762048.763864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507586816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762048.763883:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762048.763896:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762048/real 1789762048] req@ffff8d25d25bb800 x1876699507586304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762048.763907:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507586304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762048.763915:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762048.763918:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762048.763921:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507586304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762048.763927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507586432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762048.763942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762048.763950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762048/real 1789762048] req@ffff8d25d25ba680 x1876699507586432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762048.763959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507586432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762048.763978:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762048.763981:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762048.763983:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507586432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762048.763987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507586560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762048.763999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762048.764006:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762048/real 1789762048] req@ffff8d25d25bad80 x1876699507586560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762048.764013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507586560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762048.764018:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762048.764020:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762048.764022:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507586560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762048.764026:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507586688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762048.764036:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762048.764042:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762048/real 1789762048] req@ffff8d25d25bbb80 x1876699507586688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762048.764049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507586688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762048.764054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762048.764056:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762048.764058:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507586688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762048.764069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507586816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762048.764080:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762048.764087:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762048/real 1789762048] req@ffff8d25d25b9180 x1876699507586816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762048.764094:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507586816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762073 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762048.764099:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762048.764100:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762048.764102:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507586816:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762048.827546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762049.851508:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762050.875555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762051.899553:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762052.923603:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762053.883489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762053.883499:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762053.883504:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762053.883515:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2031 00000100:00080000:3.0:1789762053.883525:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762053.883533:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762053.883574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507586944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762053.883602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762053.883608:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762053.883610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762053.883614:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2031 00000100:00080000:3.0:1789762053.883619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762053.883623:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762053.883633:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762053.883652:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507587072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762053.883664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762053.883669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762053.883671:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762053.883677:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2031 00000100:00080000:3.0:1789762053.883683:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762053.883689:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762053.883696:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762053.883712:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507586944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762053.883714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507587200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762053.883779:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762053.883796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762053.883800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762053.883803:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762053.883808:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2031 00000100:00080000:3.0:1789762053.883813:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762053.883821:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762053.883823:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762053/real 1789762053] req@ffff8d25d1b0b480 x1876699507586944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762053.883830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762053.883850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507587328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762053.883854:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507586944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762053.883877:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762053.883878:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762053.883884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762053.883885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762053.883890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762053.883891:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507586944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762053.883899:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2031 00000100:00100000:0.0:1789762053.883902:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762053.883908:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762053.883916:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762053.883922:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762053.883927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762053.883944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762053/real 1789762053] req@ffff8d25d1b09500 x1876699507587072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762053.883944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507587456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762053.883981:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507587072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762053.883988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762053.883991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762053.883993:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762053.884012:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762053.884025:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762053.884032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762053/real 1789762053] req@ffff8d25d1b08000 x1876699507587200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762053.884040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507587200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762053.884045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762053.884047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762053.884049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762053.884052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762053.884066:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762053.884074:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762053/real 1789762053] req@ffff8d25d1b08380 x1876699507587328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762053.884082:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507587328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762053.884088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762053.884090:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762053.884093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587328:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789762053.884099:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762053.884104:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762053.884106:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507587456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762053.884107:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762053.884117:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2031 00000100:00080000:3.0:1789762053.884121:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762053.884125:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762053.884130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789762053.884135:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762053.884144:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762053/real 1789762053] req@ffff8d25d1b0a300 x1876699507587456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762053.884154:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507587456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762053.884160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762053.884163:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762053.884166:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507587456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762053.884200:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507587584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762053.884232:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507587584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762053.884251:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762053.884260:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762053/real 1789762053] req@ffff8d25d1b08380 x1876699507587584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762053.884273:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507587584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762078 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762053.884278:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762053.884280:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762053.884282:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507587584:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762053.947538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762054.971524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762055.995511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762057.019525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762058.043554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762059.003492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.003501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762059.003505:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.003515:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2036 00000100:00080000:2.0:1789762059.003524:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.003530:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762059.003566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507587712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762059.003597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.003600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762059.003602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.003607:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2036 00000100:00080000:2.0:1789762059.003610:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.003614:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762059.003623:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762059.003638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507587840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762059.003648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.003652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762059.003654:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.003659:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2036 00000100:00100000:0.0:1789762059.003661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507587712:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762059.003663:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.003671:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762059.003685:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762059.003699:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507587968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762059.003709:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.003712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762059.003715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.003719:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2036 00000100:00080000:2.0:1789762059.003723:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762059.003730:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762059.003730:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762059.003744:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762059.003757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507588096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762059.003768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762059.003771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762059.003773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762059.003777:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffff8d25c8020a80 x1876699507587712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762059.003781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2036 00000100:00080000:2.0:1789762059.003788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.003795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762059.003800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762059.003801:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507587712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762059.003812:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789762059.003814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507588224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762059.003817:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762059.003825:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507587712:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762059.003834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762059.003838:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587840:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762059.003841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762059.003845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762059.003850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2036 00000100:00080000:2.0:1789762059.003855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762059.003859:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762059.003864:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762059.003866:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762059.003876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffff8d25c8021180 x1876699507587840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762059.003881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507588352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762059.003892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507587840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762059.003906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762059.003908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762059.003911:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762059.003929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762059.003941:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762059.003948:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffff8d25c8022680 x1876699507587968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762059.003956:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507587968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762059.003961:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762059.003963:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762059.003965:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507587968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762059.003969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762059.003978:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762059.003984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffff8d25c8022a00 x1876699507588096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762059.003991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507588096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762059.003996:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762059.003998:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762059.004000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762059.004004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507588224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762059.004014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762059.004020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffff8d25c8023800 x1876699507588224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762059.004027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507588224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762059.004032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762059.004033:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762059.004035:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507588224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762059.004048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507588352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762059.004060:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762059.004067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762059/real 1789762059] req@ffff8d25c8020000 x1876699507588352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762059.004078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507588352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762084 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762059.004086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762059.004089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762059.004093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507588352:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762059.067540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762060.091532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762061.115495:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762062.139546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762063.163542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762064.123464:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762064.123475:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762064.123480:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762064.123491:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2041 00000100:00080000:2.0:1789762064.123501:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762064.123509:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762064.123545:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507588480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762064.123572:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762064.123577:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762064.123579:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762064.123584:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2041 00000100:00080000:2.0:1789762064.123589:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762064.123593:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762064.123603:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762064.123617:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507588608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762064.123622:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507588480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762064.123637:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762064.123642:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762064.123645:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762064.123651:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2041 00000100:00080000:2.0:1789762064.123656:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762064.123660:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762064.123666:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762064.123678:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507588736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762064.123689:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762064.123692:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762064.123694:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762064.123699:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2041 00000100:00080000:2.0:1789762064.123703:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762064.123708:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762064.123715:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762064.123730:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507588864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762064.123733:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588608:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762064.123751:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762064.123755:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762064.123758:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762064.123764:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2041 00000100:00080000:2.0:1789762064.123768:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762064.123773:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762064.123779:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588736:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789762064.123779:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762064.123799:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507588992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762064.123811:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762064.123815:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762064.123818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588864:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1789762064.123819:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762064.123828:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2041 00000100:00080000:2.0:1789762064.123833:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762064.123838:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762064.123844:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762064.123856:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507588992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762064.123860:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507589120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762064.123890:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507589120:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762064.187570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762065.211536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762066.235558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789762067.195653:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789762067.195697:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789762067.195721:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762067.195766:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762067.195777:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762067.195784:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762067.195795:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762067.195803:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789762067.195887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762067] req@ffff8d25c8022300 x1876699507589120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762067.195914:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507589120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762067.195928:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762067.195933:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762067.195938:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507589120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762067.195946:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762067] req@ffff8d25c8020380 x1876699507588864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762067.195961:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507588864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762067.195971:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762067.195975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762067.195979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762067.195986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762067] req@ffff8d25c8022d80 x1876699507588992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762067.196008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507588992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762067.196017:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762067.196020:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762067.196024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507588992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762067.196031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762067] req@ffff8d25c8023100 x1876699507588608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762067.196044:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507588608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762067.196054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762067.196058:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762067.196061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762067.196083:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762067] req@ffff8d25c8023b80 x1876699507588736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762067.196097:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507588736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762067.196106:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762067.196110:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762067.196113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507588736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762067.196120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762064/real 1789762067] req@ffff8d25c8020700 x1876699507588480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762067.196135:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507588480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762089 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762067.196144:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762067.196148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762067.196152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507588480:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1789762067.259540:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762068.283561:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762069.243504:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762069.243512:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762069.243516:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762069.243525:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2046 00000100:00080000:1.0:1789762069.243533:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762069.243539:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762069.243573:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507589248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762069.243601:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762069.243605:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762069.243607:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762069.243612:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2046 00000100:00080000:1.0:1789762069.243616:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762069.243620:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762069.243645:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762069.243661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507589248:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762069.243665:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507589376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762069.243680:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762069.243684:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762069.243686:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762069.243690:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2046 00000100:00080000:1.0:1789762069.243695:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762069.243699:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762069.243713:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762069.243728:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762069.243737:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507589504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762069.243750:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762069.243755:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762069.243757:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762069.243762:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2046 00000100:00080000:1.0:1789762069.243767:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762069.243772:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762069.243779:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762069.243785:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffff8d25c8168000 x1876699507589248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762069.243796:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507589632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762069.243810:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762069.243812:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507589248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762069.243818:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762069.243823:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762069.243832:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2046 00000100:00080000:0.0:1789762069.243835:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762069.243838:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762069.243843:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762069.243849:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762069.243854:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507589248:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789762069.243858:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762069.243880:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507589760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762069.243882:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507589376:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762069.243903:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762069.243907:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762069.243909:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762069.243915:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2046 00000100:00080000:1.0:1789762069.243919:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762069.243920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762069.243928:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762069.243936:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762069.243943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffff8d25c8169500 x1876699507589376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762069.243953:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507589888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762069.243966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507589376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.243977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.243982:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762069.243987:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507589376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762069.243995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507589504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762069.244016:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762069.244028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffff8d25c8168700 x1876699507589504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762069.244041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507589504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.244048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.244066:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762069.244071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507589504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762069.244077:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507589632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762069.244098:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762069.244110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffff8d25c8168380 x1876699507589632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762069.244123:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507589632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.244132:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.244135:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762069.244138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507589632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762069.244154:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507589760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762069.244174:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762069.244185:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffff8d25c8168e00 x1876699507589760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762069.244197:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507589760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.244205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.244208:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762069.244211:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507589760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762069.244218:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507589888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762069.244236:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762069.244248:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762069/real 1789762069] req@ffff8d25c8169f80 x1876699507589888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762069.244260:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507589888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762094 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762069.244276:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762069.244280:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762069.244283:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507589888:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762069.307569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762070.331580:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762071.355577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762072.379534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762073.403576:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762074.363528:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762074.363537:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762074.363542:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762074.363553:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2052 00000100:00080000:1.0:1789762074.363562:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762074.363574:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762074.363609:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507590016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762074.363627:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762074.363630:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762074.363631:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762074.363635:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2052 00000100:00080000:1.0:1789762074.363637:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762074.363640:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762074.363648:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762074.363657:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507590144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762074.363665:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762074.363668:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507590016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762074.363668:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762074.363672:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762074.363681:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2052 00000100:00080000:1.0:1789762074.363685:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762074.363690:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762074.363695:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762074.363711:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507590272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762074.363722:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762074.363724:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762074.363726:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762074.363732:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2052 00000800:00000100:0.0:1789762074.363734:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762074.363743:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762074.363747:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762074.363752:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762074.363762:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507590400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762074.363770:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762074.363773:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762074.363774:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762074.363778:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffff8d25c816a680 x1876699507590016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762074.363779:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2052 00000100:00080000:1.0:1789762074.363784:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762074.363789:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762074.363796:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762074.363806:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507590528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762074.363820:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762074.363823:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507590016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762074.363824:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762074.363827:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762074.363835:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2052 00000100:00080000:1.0:1789762074.363840:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762074.363843:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789762074.363845:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:1.0:1789762074.363847:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762074.363852:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762074.363856:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507590656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762074.363859:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507590016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762074.363891:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762074.363915:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762074.363927:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffff8d25c816aa00 x1876699507590144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762074.363940:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507590144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762074.363951:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762074.363954:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762074.363958:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762074.363964:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762074.363980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762074.363990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffff8d25c8169f80 x1876699507590272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762074.364002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507590272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762074.364032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762074.364036:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762074.364111:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762074.364120:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762074.364144:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762074.364156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffff8d25c8168e00 x1876699507590400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762074.364166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507590400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762074.364173:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762074.364176:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762074.364178:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762074.364182:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507590528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762074.364194:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762074.364201:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffff8d25c8168380 x1876699507590528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762074.364209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507590528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762074.364214:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762074.364216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762074.364218:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507590528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762074.364222:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507590656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762074.364232:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762074.364238:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762074/real 1789762074] req@ffff8d25c8168700 x1876699507590656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762074.364245:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507590656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762099 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762074.364250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762074.364252:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762074.364253:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507590656:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762074.427555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762075.451542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762076.475568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762077.499582:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762078.523545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762079.483518:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762079.483525:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762079.483529:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762079.483538:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2057 00000100:00080000:3.0:1789762079.483546:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762079.483553:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762079.483589:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507590784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762079.483616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762079.483619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762079.483621:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762079.483624:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2057 00000100:00080000:3.0:1789762079.483627:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762079.483631:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762079.483640:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762079.483653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507590912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762079.483655:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507590784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762079.483669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762079.483671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762079.483672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762079.483676:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2057 00000100:00080000:3.0:1789762079.483678:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762079.483680:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762079.483684:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762079.483692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507591040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762079.483698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762079.483700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762079.483701:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762079.483704:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2057 00000100:00080000:3.0:1789762079.483706:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762079.483708:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762079.483712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762079.483715:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762079.483722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507591168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762079.483730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762079.483732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762079.483734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762079.483736:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2057 00000100:00080000:3.0:1789762079.483739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762079.483742:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762079.483745:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762079.483753:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762079/real 1789762079] req@ffff8d25d1b08000 x1876699507590784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762079.483755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507591296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762079.483786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762079.483788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762079.483790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762079.483794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2057 00000100:00100000:0.0:1789762079.483795:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507590784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762079.483797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762079.483803:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762079.483809:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762079.483815:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789762079.483820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507591424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762079.483821:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762079.483828:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507590784:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762079.483883:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762079.483921:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762079.483935:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762079/real 1789762079] req@ffff8d25d1b09500 x1876699507590912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762079.483949:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507590912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762079.483957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762079.483960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762079.483964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507590912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762079.483970:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762079.484030:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762079.484041:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762079/real 1789762079] req@ffff8d25d1b0b480 x1876699507591040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762079.484052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507591040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762079.484060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762079.484063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762079.484066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762079.484071:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762079.484090:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762079.484101:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762079/real 1789762079] req@ffff8d25d1b0b100 x1876699507591168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762079.484114:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507591168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762079.484119:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762079.484121:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762079.484130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762079.484135:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507591296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762079.484153:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762079.484161:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762079/real 1789762079] req@ffff8d25d1b09c00 x1876699507591296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762079.484168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507591296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762079.484174:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762079.484176:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762079.484178:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507591296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762079.484181:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507591424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762079.484191:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762079.484197:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762079/real 1789762079] req@ffff8d25d1b0ad80 x1876699507591424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762079.484203:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507591424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762104 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762079.484208:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762079.484210:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762079.484212:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507591424:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762079.547539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762080.571558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762081.595620:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762082.619514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762083.643574:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762084.603516:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762084.603524:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762084.603528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762084.603537:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2062 00000100:00080000:3.0:1789762084.603546:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762084.603552:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762084.603599:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507591552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762084.603620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762084.603623:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762084.603624:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762084.603628:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2062 00000100:00080000:3.0:1789762084.603632:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762084.603646:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762084.603654:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762084.603662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507591680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762084.603669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762084.603672:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762084.603674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762084.603676:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2062 00000100:00080000:3.0:1789762084.603678:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762084.603681:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762084.603684:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762084.603690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507591808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762084.603702:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507591552:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762084.603705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762084.603720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762084.603722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762084.603726:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2062 00000100:00080000:3.0:1789762084.603728:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762084.603730:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762084.603733:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762084.603744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507591936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762084.603751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762084.603754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762084.603755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762084.603757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2062 00000100:00080000:3.0:1789762084.603760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762084.603762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762084.603766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762084.603773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507592064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762084.603779:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762084.603783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762084.603785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762084.603787:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2062 00000100:00080000:3.0:1789762084.603790:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762084.603792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762084.603795:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762084.603800:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762084.603809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507592192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762084.603879:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762084/real 1789762084] req@ffff8d25d1b09180 x1876699507591552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762084.603898:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507591552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762084.603908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762084.603912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762084.603932:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507591552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762084.603954:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762084.603982:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762084.603994:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762084/real 1789762084] req@ffff8d25d1b0aa00 x1876699507591680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762084.604003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507591680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762084.604009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762084.604012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762084.604014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762084.604044:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762084.604058:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762084.604067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762084/real 1789762084] req@ffff8d25d1b08a80 x1876699507591808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762084.604075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507591808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762084.604081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762084.604083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762084.604086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762084.604090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762084.604101:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762084.604107:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762084/real 1789762084] req@ffff8d25d1b0bb80 x1876699507591936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762084.604115:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507591936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762084.604128:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762084.604130:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762084.604132:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507591936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762084.604136:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762084.604149:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762084.604156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762084/real 1789762084] req@ffff8d25d1b09880 x1876699507592064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762084.604164:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507592064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762084.604170:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762084.604173:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762084.604176:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762084.604183:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762084.604212:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762084.604225:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762084/real 1789762084] req@ffff8d25d1b0a680 x1876699507592192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762084.604237:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507592192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762109 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762084.604245:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762084.604249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762084.604252:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592192:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762084.667532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762085.691533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762086.715550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762087.739610:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762088.763635:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762089.723466:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762089.723473:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762089.723477:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762089.723486:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2067 00000100:00080000:3.0:1789762089.723495:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762089.723501:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762089.723533:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507592320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762089.723557:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762089.723559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762089.723561:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762089.723564:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2067 00000100:00080000:3.0:1789762089.723566:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762089.723569:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762089.723576:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762089.723583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507592448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762089.723590:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762089.723592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762089.723594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762089.723597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2067 00000100:00080000:3.0:1789762089.723599:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762089.723601:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762089.723604:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762089.723610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507592576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762089.723615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762089.723617:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762089.723619:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762089.723622:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2067 00000100:00080000:3.0:1789762089.723624:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762089.723626:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762089.723629:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762089.723636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507592704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762089.723645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762089.723652:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789762089.723654:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507592320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762089.723654:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762089.723668:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2067 00000100:00080000:3.0:1789762089.723671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762089.723674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762089.723678:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762089.723686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507592832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762089.723692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762089.723695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762089.723697:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762089.723699:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2067 00000100:00080000:3.0:1789762089.723702:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762089.723704:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762089.723707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762089.723715:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507592960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762089.723755:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762089.723832:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762089/real 1789762089] req@ffff8d25d1b08e00 x1876699507592320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762089.723851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507592320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762089.723864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762089.723870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762089.723890:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507592320:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762089.723923:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507592448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762089.723960:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762089.723996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762089/real 1789762089] req@ffff8d25d1b0b800 x1876699507592448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762089.724008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507592448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762089.724016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762089.724020:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762089.724024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507592448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762089.724046:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507592576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762089.724062:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762089.724069:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762089/real 1789762089] req@ffff8d25d1b08700 x1876699507592576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762089.724079:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507592576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762089.724087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762089.724089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762089.724092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507592576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762089.724160:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507592704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762089.724200:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762089.724212:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762089/real 1789762089] req@ffff8d25d1b0a680 x1876699507592704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762089.724221:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507592704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762089.724228:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762089.724232:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762089.724250:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507592704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762089.724259:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762089.724278:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762089.724288:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762089/real 1789762089] req@ffff8d25d1b09880 x1876699507592832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762089.724298:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507592832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762089.724306:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762089.724309:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762089.724314:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762089.724340:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762089.724433:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762089.724449:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762089/real 1789762089] req@ffff8d25d1b0bb80 x1876699507592960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762089.724496:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507592960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762114 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762089.724508:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762089.724512:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762089.724518:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507592960:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762089.787535:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762090.811624:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762091.835562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762092.859563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762093.883527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762094.843512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762094.843522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762094.843525:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762094.843535:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2072 00000100:00080000:3.0:1789762094.843544:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762094.843550:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762094.843587:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507593088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762094.843609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762094.843614:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762094.843617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762094.843625:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2072 00000100:00080000:3.0:1789762094.843628:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762094.843631:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762094.843643:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762094.843656:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507593216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762094.843663:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762094.843666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762094.843667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762094.843670:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2072 00000100:00080000:3.0:1789762094.843671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762094.843674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762094.843676:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762094.843684:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507593344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762094.843692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762094.843695:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762094.843698:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762094.843706:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2072 00000100:00080000:3.0:1789762094.843709:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762094.843710:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507593088:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762094.843722:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762094.843732:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762094.843742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507593472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762094.843750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762094.843752:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762094.843753:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762094.843759:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2072 00000100:00080000:3.0:1789762094.843762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762094.843765:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762094.843768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762094.843780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507593600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762094.843789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762094.843792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762094.843794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762094.843799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2072 00000100:00080000:3.0:1789762094.843803:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762094.843807:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762094.843817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762094.843830:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762094.843837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507593728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762094.843924:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762094/real 1789762094] req@ffff8d25d1b08a80 x1876699507593088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762094.843951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507593088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762094.843965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762094.843970:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762094.843975:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507593088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762094.843987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762094.844018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762094.844034:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762094/real 1789762094] req@ffff8d25d1b0aa00 x1876699507593216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762094.844048:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507593216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762094.844059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762094.844063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762094.844067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762094.844107:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762094.844129:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762094.844142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762094/real 1789762094] req@ffff8d25d1b09180 x1876699507593344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762094.844169:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507593344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762094.844179:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762094.844183:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762094.844187:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762094.844194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762094.844215:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762094.844227:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762094/real 1789762094] req@ffff8d25d1b0ad80 x1876699507593472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762094.844240:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507593472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762094.844250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762094.844253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762094.844256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762094.844263:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507593600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762094.844282:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762094.844293:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762094/real 1789762094] req@ffff8d25d1b09c00 x1876699507593600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762094.844307:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507593600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762094.844335:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762094.844339:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762094.844343:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507593600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762094.844350:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507593728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762094.844373:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762094.844385:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762094/real 1789762094] req@ffff8d25d1b0b100 x1876699507593728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762094.844451:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507593728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762119 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762094.844468:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762094.844475:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762094.844481:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507593728:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762094.907609:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762095.931527:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762096.955558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762097.979500:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762099.003501:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762099.963634:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762099.963647:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762099.963653:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762099.963665:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2077 00000100:00080000:0.0:1789762099.963678:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762099.963688:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762099.963736:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507593856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762099.963778:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762099.963783:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762099.963785:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762099.963791:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2077 00000100:00080000:0.0:1789762099.963796:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762099.963802:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762099.963815:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762099.963864:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507593856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762099.963867:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507593984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762099.963898:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762099.963905:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762099.963910:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762099.963918:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2077 00000100:00080000:0.0:1789762099.963928:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762099.963938:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762099.963950:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762099.963971:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762099.964000:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507594112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762099.964025:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762099.964033:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762099.964039:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762099.964044:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762099/real 1789762099] req@ffff8d25d25b8a80 x1876699507593856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762099.964051:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2077 00000100:00080000:0.0:1789762099.964063:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762099.964078:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762099.964092:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762099.964106:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507593856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762099.964118:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507594240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762099.964156:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762099.964159:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762099.964169:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762099.964170:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762099.964180:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762099.964188:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507593856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762099.964195:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2077 00000100:00080000:0.0:1789762099.964204:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762099.964211:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762099.964218:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762099.964232:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507594368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762099.964239:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593984:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762099.964265:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762099.964271:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762099.964275:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762099.964283:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2077 00000100:00080000:0.0:1789762099.964289:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762099.964296:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762099.964300:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762099.964311:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762099.964326:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762099/real 1789762099] req@ffff8d25d25ba300 x1876699507593984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762099.964335:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507594496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762099.964366:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507593984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762099.964423:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762099.964428:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762099.964434:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507593984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762099.964446:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762099.964506:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762099.964533:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762099/real 1789762099] req@ffff8d25d25b8000 x1876699507594112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762099.964570:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507594112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762099.964581:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762099.964585:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762099.964589:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762099.964597:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762099.964626:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762099.964641:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762099/real 1789762099] req@ffff8d25d25b9180 x1876699507594240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762099.964655:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507594240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762099.964665:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762099.964669:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762099.964672:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762099.964693:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507594368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762099.964715:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762099.964728:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762099/real 1789762099] req@ffff8d25d25bbb80 x1876699507594368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762099.964741:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507594368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762099.964759:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762099.964763:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762099.964767:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507594368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762099.964773:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507594496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762099.964791:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762099.964803:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762099/real 1789762099] req@ffff8d25d25bad80 x1876699507594496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762099.964817:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507594496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762124 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762099.964857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762099.964861:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762099.964865:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507594496:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762100.027598:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762101.051633:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762102.075557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762103.099526:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762104.123513:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762105.083510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762105.083522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762105.083528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762105.083544:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2082 00000100:00080000:3.0:1789762105.083556:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762105.083564:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762105.083609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507594624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762105.083639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762105.083642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762105.083644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762105.083647:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2082 00000100:00080000:3.0:1789762105.083650:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762105.083653:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762105.083663:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762105.083676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507594752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762105.083677:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507594624:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762105.083690:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762105.083693:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762105.083694:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762105.083698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2082 00000100:00080000:3.0:1789762105.083700:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762105.083702:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762105.083705:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762105.083713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507594880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762105.083719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762105.083721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762105.083723:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762105.083726:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2082 00000100:00080000:3.0:1789762105.083730:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762105.083733:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762105.083737:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762105.083744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507595008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762105.083751:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762105.083754:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762105.083755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762105.083757:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2082 00000100:00080000:3.0:1789762105.083760:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762105.083762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762105.083768:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789762105.083769:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762105.083778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507595136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762105.083801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762105.083807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762105.083809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762105.083814:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2082 00000100:00080000:3.0:1789762105.083819:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762105.083823:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762105.083829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762105.083840:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffff8d25d1b0b480 x1876699507594624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762105.083847:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507595264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762105.083875:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507594624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.083890:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.083894:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762105.083898:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507594624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762105.083923:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762105.083945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762105.083955:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffff8d25d1b09500 x1876699507594752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.083965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507594752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.083971:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.083974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762105.083976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762105.083980:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762105.083991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762105.083999:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffff8d25d1b08000 x1876699507594880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.084006:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507594880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.084012:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.084014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762105.084018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507594880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762105.084024:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762105.084045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762105.084059:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffff8d25d1b08380 x1876699507595008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.084072:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507595008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.084079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.084081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762105.084083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762105.084088:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507595136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762105.084104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762105.084121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffff8d25d1b0a300 x1876699507595136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.084129:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507595136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.084138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.084141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762105.084145:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507595136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762105.084151:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507595264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762105.084172:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762105.084184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762105/real 1789762105] req@ffff8d25d1b0b100 x1876699507595264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762105.084194:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507595264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762130 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762105.084200:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762105.084202:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762105.084204:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507595264:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762105.147558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762106.171578:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762107.195531:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762108.219635:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762109.243484:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762110.203520:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762110.203531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762110.203553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762110.203567:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2087 00000100:00080000:3.0:1789762110.203581:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762110.203592:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762110.203640:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507595392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762110.203674:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762110.203679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762110.203682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762110.203687:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2087 00000100:00080000:3.0:1789762110.203693:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762110.203699:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762110.203712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762110.203725:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507595392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762110.203734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507595520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762110.203786:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762110.203794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762110.203798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762110.203808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2087 00000100:00080000:3.0:1789762110.203818:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762110.203827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762110.203839:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762110.203850:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762110.203875:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507595648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762110.203901:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762110.203909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffff8d25d1b09c00 x1876699507595392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762110.203910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762110.203916:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762110.203932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2087 00000100:00080000:3.0:1789762110.203943:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762110.203950:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762110.203955:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507595392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762110.203962:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1789762110.203981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762110.203990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762110.203995:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507595776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762110.204002:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507595392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762110.204029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762110.204034:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762110.204037:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762110.204041:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2087 00000100:00080000:3.0:1789762110.204045:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762110.204049:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762110.204055:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762110.204064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762110.204075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507595904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762110.204094:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762110.204099:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762110.204102:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762110.204111:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762110.204112:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2087 00000100:00080000:3.0:1789762110.204126:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762110.204133:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762110.204141:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762110.204147:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffff8d25d1b0ad80 x1876699507595520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762110.204164:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507596032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762110.204173:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507595520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762110.204227:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762110.204234:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762110.204242:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762110.204254:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762110.204289:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762110.204304:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffff8d25d1b09180 x1876699507595648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762110.204335:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507595648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762110.204346:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762110.204350:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762110.204354:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762110.204362:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762110.204386:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762110.204399:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffff8d25d1b0aa00 x1876699507595776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762110.204462:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507595776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762110.204498:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762110.204503:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762110.204509:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507595776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762110.204539:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507595904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762110.204583:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762110.204600:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffff8d25d1b08a80 x1876699507595904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762110.204615:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507595904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762110.204625:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762110.204629:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762110.204633:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507595904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762110.204639:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507596032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762110.204659:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762110.204670:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762110/real 1789762110] req@ffff8d25d1b0bb80 x1876699507596032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762110.204684:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507596032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762135 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762110.204694:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762110.204698:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762110.204702:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507596032:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762110.267619:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762111.291518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762112.315694:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762113.339578:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762114.363616:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762115.323467:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.323475:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762115.323478:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.323487:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2093 00000100:00080000:0.0:1789762115.323507:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.323514:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762115.323554:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507596160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762115.323578:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.323582:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762115.323584:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.323589:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2093 00000100:00080000:0.0:1789762115.323593:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.323598:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762115.323607:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762115.323612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507596160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762115.323620:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507596288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762115.323629:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.323632:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762115.323635:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.323639:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2093 00000100:00080000:0.0:1789762115.323641:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.323645:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762115.323649:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762115.323657:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507596416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762115.323665:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.323668:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789762115.323669:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762115.323670:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.323674:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2093 00000100:00080000:0.0:1789762115.323676:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.323679:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762115.323684:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762115.323693:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507596544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762115.323702:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.323705:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762115.323706:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762115.323721:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffff8d25d25ba680 x1876699507596160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762115.323722:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2093 00000100:00080000:0.0:1789762115.323726:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.323730:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762115.323746:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762115.323751:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507596160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.323762:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762115.323763:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507596672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762115.323766:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762115.323770:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507596160:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762115.323773:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762115.323776:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762115.323778:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762115.323781:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2093 00000100:00080000:0.0:1789762115.323784:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762115.323787:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762115.323791:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762115.323798:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507596288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762115.323800:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507596800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762115.323832:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762115.323846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffff8d25d25bb800 x1876699507596288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.323860:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507596288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.323866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.323868:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762115.323871:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507596288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762115.323875:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507596416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762115.323891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762115.323898:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffff8d25d25b9f80 x1876699507596416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.323905:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507596416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.323910:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.323912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762115.323914:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507596416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762115.323917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507596544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762115.323928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762115.323934:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffff8d25d25b8380 x1876699507596544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.323946:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507596544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.323951:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.323953:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762115.323955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507596544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762115.323958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507596672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762115.323968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762115.323974:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffff8d25d25b8700 x1876699507596672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.323981:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507596672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.323986:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.323988:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762115.323990:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507596672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762115.324001:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507596800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762115.324012:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762115.324018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762115/real 1789762115] req@ffff8d25d25bb100 x1876699507596800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762115.324035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507596800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762140 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762115.324055:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762115.324057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762115.324059:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507596800:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762115.387565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762116.411617:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762117.435494:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762118.459600:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762119.483551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762120.443614:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762120.443645:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762120.443651:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762120.443663:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2098 00000100:00080000:2.0:1789762120.443694:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762120.443704:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762120.443773:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507596928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762120.443804:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762120.443808:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762120.443810:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762120.443831:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2098 00000100:00080000:2.0:1789762120.443834:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762120.443839:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762120.443851:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762120.443866:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507597056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762120.443878:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762120.443883:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762120.443886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507596928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762120.443888:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762120.443904:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2098 00000100:00080000:2.0:1789762120.443908:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762120.443911:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762120.443916:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762120.443930:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507597184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762120.443940:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762120.443943:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762120.443946:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762120.443950:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2098 00000100:00080000:2.0:1789762120.443961:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762120.443986:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762120.443987:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762120.443999:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762120.444016:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507597312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762120.444050:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762120.444054:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762120.444058:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762120.444064:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2098 00000100:00080000:2.0:1789762120.444071:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762120.444077:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762120/real 1789762120] req@ffff8d25c8021880 x1876699507596928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762120.444086:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762120.444118:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507596928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762120.444119:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762120.444140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762120.444151:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762120.444153:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507597440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762120.444157:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507596928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762120.444185:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762120.444188:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762120.444191:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762120.444197:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2098 00000100:00080000:2.0:1789762120.444202:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762120.444202:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597056:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762120.444219:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762120.444227:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762120.444237:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762120.444250:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507597568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762120.444262:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762120/real 1789762120] req@ffff8d25c8023480 x1876699507597056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762120.444290:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507597056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762120.444311:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762120.444316:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762120.444319:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762120.444325:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762120.444358:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762120.444369:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762120/real 1789762120] req@ffff8d25c8020e00 x1876699507597184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762120.444423:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507597184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762120.444430:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762120.444433:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762120.444436:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762120.444441:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762120.444459:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762120.444468:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762120/real 1789762120] req@ffff8d25c8021500 x1876699507597312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762120.444477:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507597312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762120.444483:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762120.444486:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762120.444489:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762120.444494:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507597440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762120.444514:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762120.444527:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762120/real 1789762120] req@ffff8d25c8021c00 x1876699507597440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762120.444542:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507597440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762120.444553:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762120.444558:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762120.444562:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507597440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762120.444583:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507597568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762120.444602:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762120.444612:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762120/real 1789762120] req@ffff8d25c8021880 x1876699507597568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762120.444622:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507597568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762145 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762120.444628:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762120.444631:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762120.444634:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507597568:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762120.507581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762121.531649:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762122.555615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762123.579674:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762124.603592:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762125.563552:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762125.563562:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762125.563566:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762125.563577:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2103 00000100:00080000:1.0:1789762125.563587:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762125.563595:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762125.563669:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507597696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762125.563704:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762125.563708:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762125.563712:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762125.563733:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2103 00000100:00080000:1.0:1789762125.563738:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762125.563744:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762125.563767:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762125.563782:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507597824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762125.563795:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507597696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762125.563797:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762125.563807:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762125.563809:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762125.563814:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2103 00000100:00080000:1.0:1789762125.563817:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762125.563824:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762125.563831:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762125.563852:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507597952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762125.563862:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762125.563866:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762125.563868:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762125.563871:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2103 00000100:00080000:1.0:1789762125.563877:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762125.563881:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762125.563887:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789762125.563892:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762125.563911:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507598080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762125.563937:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762125.563940:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762125.563944:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762125.563949:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2103 00000100:00080000:1.0:1789762125.563956:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762125.563964:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762125.563965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762125/real 1789762125] req@ffff8d25c8169500 x1876699507597696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762125.563971:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762125.564005:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507598208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762125.564013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507597696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762125.564018:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762125.564027:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762125.564031:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762125.564038:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762125.564039:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2103 00000100:00080000:1.0:1789762125.564047:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762125.564050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762125.564062:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507597696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762125.564064:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762125.564078:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762125.564087:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507598336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762125.564092:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762125.564130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762125.564147:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762125/real 1789762125] req@ffff8d25c8168000 x1876699507597824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762125.564158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507597824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762125.564166:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762125.564168:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762125.564171:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762125.564186:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762125.564203:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762125.564212:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762125/real 1789762125] req@ffff8d25c816b480 x1876699507597952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762125.564221:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507597952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762125.564230:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762125.564233:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762125.564235:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507597952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762125.564240:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762125.564253:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762125.564263:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762125/real 1789762125] req@ffff8d25c816a300 x1876699507598080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762125.564285:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507598080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762125.564296:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762125.564300:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762125.564304:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762125.564312:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507598208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762125.564338:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762125.564355:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762125/real 1789762125] req@ffff8d25c816ad80 x1876699507598208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762125.564368:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507598208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762125.564406:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762125.564411:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762125.564414:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507598208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762125.564448:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507598336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762125.564471:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762125.564481:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762125/real 1789762125] req@ffff8d25c8168a80 x1876699507598336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762125.564490:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507598336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762150 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762125.564501:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762125.564504:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762125.564506:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507598336:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762125.627603:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762126.651617:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762127.675612:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762128.699594:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762129.723699:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762130.683486:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762130.683496:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762130.683500:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762130.683509:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2108 00000100:00080000:1.0:1789762130.683517:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762130.683523:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762130.683571:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507598464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762130.683608:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762130.683611:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762130.683613:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762130.683617:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2108 00000100:00080000:1.0:1789762130.683620:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762130.683624:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762130.683632:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762130.683641:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507598592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762130.683650:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762130.683650:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507598464:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762130.683657:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762130.683660:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762130.683664:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2108 00000100:00080000:1.0:1789762130.683666:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762130.683668:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762130.683672:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762130.683681:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507598720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762130.683688:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762130.683690:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762130.683692:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762130.683694:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2108 00000100:00080000:1.0:1789762130.683697:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762130.683711:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762130.683715:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762130.683722:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507598848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762130.683729:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762130.683731:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762130.683733:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762130.683735:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2108 00000100:00080000:1.0:1789762130.683737:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762130.683739:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762130.683742:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762130.683749:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507598976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762130.683755:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762130.683759:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762130.683760:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762130.683766:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2108 00000100:00100000:2.0:1789762130.683775:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762130.683777:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762130.683783:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762130.683789:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762130.683801:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507599104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762130.683811:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762130.683845:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762130.683872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507598976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762130.683887:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507599104:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762130.747613:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762131.771583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762132.795518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789762133.755756:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789762133.755821:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762133.755841:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762133.755887:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762133.755895:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762133.755907:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762133.755921:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762133.755938:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789762133.755961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762130/real 1789762133] req@ffff8d25c8169c00 x1876699507598592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762133.756003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507598592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762133.756026:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762133.756043:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762133.756048:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762133.756058:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762130/real 1789762133] req@ffff8d25c8168a80 x1876699507598720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762133.756078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507598720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762133.756097:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762133.756102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762133.756105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762133.756113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762130/real 1789762133] req@ffff8d25c816ad80 x1876699507598848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762133.756127:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507598848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762133.756136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762133.756141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762133.756144:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507598848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762133.756153:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762130/real 1789762133] req@ffff8d25c816a300 x1876699507598976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762133.756166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507598976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762133.756175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762133.756179:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762133.756183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507598976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762133.756191:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762130/real 1789762133] req@ffff8d25c8169880 x1876699507598464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762133.756204:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507598464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762133.756212:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762133.756216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762133.756219:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507598464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762133.756251:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762130/real 1789762133] req@ffff8d25c816b480 x1876699507599104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762133.756265:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507599104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762155 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762133.756274:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762133.756278:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762133.756281:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507599104:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762133.819505:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762134.843503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762135.803510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762135.803520:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762135.803524:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762135.803534:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2113 00000100:00080000:0.0:1789762135.803544:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762135.803551:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762135.803601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507599232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762135.803623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762135.803627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762135.803629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762135.803633:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2113 00000100:00080000:0.0:1789762135.803638:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762135.803644:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762135.803654:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762135.803661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507599232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762135.803673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507599360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762135.803685:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762135.803689:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762135.803691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762135.803696:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2113 00000100:00080000:0.0:1789762135.803701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762135.803705:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762135.803712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762135.803719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762135.803733:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507599488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762135.803745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762135.803749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762135.803751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762135.803757:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2113 00000100:00100000:2.0:1789762135.803763:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762135/real 1789762135] req@ffff8d25d25bb480 x1876699507599232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762135.803766:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762135.803776:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762135.803786:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762135.803791:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507599232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762135.803803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507599616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762135.803806:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762135.803815:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762135.803824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762135.803825:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507599232:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762135.803835:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762135.803838:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762135.803844:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2113 00000100:00080000:0.0:1789762135.803849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762135.803854:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762135.803859:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507599360:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789762135.803863:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762135.803882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507599744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762135.803893:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762135.803894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762135.803901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762135.803904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762135.803909:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2113 00000100:00080000:0.0:1789762135.803914:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762135.803917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762135/real 1789762135] req@ffff8d25d25b8e00 x1876699507599360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762135.803922:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762135.803932:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762135.803941:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507599360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762135.803950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507599872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762135.803956:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762135.803965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762135.803972:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507599360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762135.803981:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507599488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762135.804005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762135.804019:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762135/real 1789762135] req@ffff8d25d25b9c00 x1876699507599488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762135.804031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507599488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762135.804040:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762135.804043:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762135.804046:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507599488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762135.804052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507599616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762135.804091:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762135.804102:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762135/real 1789762135] req@ffff8d25d25bb100 x1876699507599616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762135.804113:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507599616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762135.804121:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762135.804123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762135.804126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507599616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762135.804142:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507599744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762135.804163:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762135.804175:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762135/real 1789762135] req@ffff8d25d25b8700 x1876699507599744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762135.804187:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507599744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762135.804195:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762135.804198:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762135.804202:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507599744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762135.804208:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507599872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762135.804227:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762135.804245:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762135/real 1789762135] req@ffff8d25d25b8380 x1876699507599872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762135.804258:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507599872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762160 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762135.804266:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762135.804270:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762135.804274:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507599872:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762135.867549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762136.891554:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762137.915643:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762138.939509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762139.963514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762140.923483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762140.923526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762140.923531:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762140.923543:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2118 00000100:00080000:0.0:1789762140.923567:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762140.923575:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762140.923631:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507600000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762140.923656:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762140.923660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762140.923662:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762140.923667:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2118 00000100:00080000:0.0:1789762140.923670:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762140.923676:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762140.923687:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762140.923698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507600128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762140.923706:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507600000:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762140.923710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762140.923722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762140.923728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762140.923733:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2118 00000100:00080000:0.0:1789762140.923736:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762140.923738:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762140.923744:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762140.923756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507600256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762140.923764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762140.923768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762140.923770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762140.923772:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2118 00000100:00080000:0.0:1789762140.923779:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762140.923781:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762140.923784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762140.923793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507600384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762140.923801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762140.923824:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762140.923825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789762140.923828:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762140.923833:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2118 00000100:00080000:0.0:1789762140.923836:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762140.923840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762140.923844:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762140.923852:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507600512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762140.923859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762140.923861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762140.923863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762140.923867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2118 00000100:00080000:0.0:1789762140.923870:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762140.923890:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762140/real 1789762140] req@ffff8d25d25b9f80 x1876699507600000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762140.923896:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762140.923901:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762140.923916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507600640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762140.923925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507600000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762140.923935:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762140.923940:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762140.923944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507600000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762140.923971:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762140.923997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762140.924012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762140/real 1789762140] req@ffff8d25d25bb800 x1876699507600128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762140.924025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507600128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762140.924031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762140.924034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762140.924036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762140.924041:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762140.924056:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762140.924064:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762140/real 1789762140] req@ffff8d25d25ba680 x1876699507600256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762140.924071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507600256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762140.924076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762140.924079:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762140.924081:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762140.924084:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762140.924095:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762140.924100:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762140/real 1789762140] req@ffff8d25d25bad80 x1876699507600384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762140.924107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507600384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762140.924113:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762140.924115:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762140.924117:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762140.924120:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507600512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762140.924130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762140.924136:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762140/real 1789762140] req@ffff8d25d25bbb80 x1876699507600512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762140.924144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507600512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762140.924149:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762140.924150:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762140.924152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507600512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762140.924155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507600640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762140.924165:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762140.924171:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762140/real 1789762140] req@ffff8d25d25b9180 x1876699507600640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762140.924178:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507600640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762165 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762140.924182:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762140.924184:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762140.924186:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507600640:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762140.987571:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762142.011583:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762143.035552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762144.059589:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762145.083524:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762146.043533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762146.043543:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762146.043548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762146.043559:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2123 00000100:00080000:3.0:1789762146.043570:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762146.043578:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762146.043621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507600768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762146.043648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762146.043651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762146.043653:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762146.043657:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2123 00000100:00080000:3.0:1789762146.043660:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762146.043665:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762146.043675:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762146.043685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507600896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762146.043695:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507600768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762146.043697:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762146.043709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762146.043713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762146.043720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2123 00000100:00080000:3.0:1789762146.043725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762146.043731:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762146.043736:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762146.043753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507601024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762146.043764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762146.043768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762146.043771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789762146.043776:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762146.043780:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2123 00000100:00080000:3.0:1789762146.043785:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762146.043790:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762146.043798:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762146.043814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507601152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762146.043828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762146.043829:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffff8d25d1b09880 x1876699507600768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762146.043845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762146.043850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762146.043858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2123 00000100:00080000:3.0:1789762146.043861:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762146.043870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762146.043876:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762146.043882:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507600768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762146.043891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507601280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762146.043895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762146.043909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762146.043911:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762146.043916:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762146.043921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762146.043922:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507600768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762146.043937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2123 00000100:00080000:3.0:1789762146.043943:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762146.043949:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762146.043955:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762146.043956:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762146.043984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507601408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762146.043993:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762146.044016:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffff8d25d1b0a680 x1876699507600896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762146.044029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507600896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762146.044036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762146.044038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762146.044041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507600896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762146.044046:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762146.044062:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762146.044075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffff8d25d1b08700 x1876699507601024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762146.044088:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507601024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762146.044104:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762146.044107:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762146.044110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762146.044118:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762146.044140:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762146.044152:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffff8d25d1b0b800 x1876699507601152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762146.044166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507601152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762146.044175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762146.044178:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762146.044182:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762146.044189:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507601280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762146.044210:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762146.044220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffff8d25d1b08e00 x1876699507601280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762146.044228:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507601280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762146.044233:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762146.044235:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762146.044238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507601280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762146.044256:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507601408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762146.044269:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762146.044276:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762146/real 1789762146] req@ffff8d25d1b09880 x1876699507601408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762146.044283:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507601408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762171 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762146.044288:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762146.044290:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762146.044292:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507601408:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762146.107597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762147.131653:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762148.155541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762149.179549:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762150.203497:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762151.163486:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762151.163494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762151.163498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762151.163506:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2128 00000100:00080000:0.0:1789762151.163513:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762151.163519:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762151.163549:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507601536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762151.163569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762151.163572:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762151.163573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762151.163577:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2128 00000100:00080000:0.0:1789762151.163579:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762151.163582:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762151.163590:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762151.163601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507601536:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762151.163602:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507601664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762151.163620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762151.163622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762151.163624:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762151.163626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2128 00000100:00080000:0.0:1789762151.163629:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762151.163631:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762151.163635:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762151.163644:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507601792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762151.163654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762151.163658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762151.163663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789762151.163663:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762151.163671:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2128 00000100:00080000:0.0:1789762151.163677:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762151.163682:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762151.163688:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762151.163699:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762151] req@ffff8d25d25b8000 x1876699507601536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762151.163705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507601920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762151.163714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762151.163722:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762151.163724:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762151.163728:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2128 00000100:00080000:0.0:1789762151.163732:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762151.163733:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507601536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762151.163739:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762151.163747:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789762151.163751:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762151.163757:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762151.163759:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507602048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762151.163763:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507601536:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762151.163777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762151.163779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762151.163781:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762151.163784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2128 00000100:00080000:0.0:1789762151.163786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762151.163788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762151.163792:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601664:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789762151.163793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762151.163805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507602176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762151.163823:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762151.163835:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762151] req@ffff8d25d25ba300 x1876699507601664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762151.163844:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507601664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762151.163850:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762151.163852:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762151.163855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762151.163858:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762151.163869:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762151.163876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762151] req@ffff8d25d25b8a80 x1876699507601792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762151.163890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507601792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762151.163895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762151.163898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762151.163900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762151.163904:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762151.163917:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762151.163928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762151] req@ffff8d25d25b9180 x1876699507601920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762151.163947:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507601920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762151.163956:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762151.163960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762151.163963:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507601920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762151.163969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762151.163989:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762151.163997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762151] req@ffff8d25d25bbb80 x1876699507602048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762151.164005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507602048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762151.164011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762151.164013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762151.164015:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762151.164027:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762151.164038:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762151.164045:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762151/real 1789762151] req@ffff8d25d25bad80 x1876699507602176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762151.164053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507602176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762176 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762151.164068:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762151.164071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762151.164074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602176:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762151.227565:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762152.251510:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762153.275504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762154.299539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762155.323558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762156.283514:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762156.283522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762156.283527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762156.283536:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2133 00000100:00080000:3.0:1789762156.283545:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762156.283551:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762156.283581:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507602304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762156.283602:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762156.283605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762156.283606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762156.283610:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2133 00000100:00080000:3.0:1789762156.283612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762156.283616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762156.283624:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762156.283633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507602432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762156.283637:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507602304:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762156.283648:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762156.283651:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762156.283653:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762156.283656:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2133 00000100:00080000:3.0:1789762156.283659:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762156.283661:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762156.283665:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762156.283677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507602560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762156.283693:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762156.283699:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762156.283707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762156.283713:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762156.283719:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2133 00000100:00080000:3.0:1789762156.283724:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762156.283729:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762156.283734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762156.283739:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffff8d25d1b0bb80 x1876699507602304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762156.283749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507602688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762156.283757:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507602304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762156.283758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762156.283767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762156.283770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762156.283779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2133 00000100:00080000:2.0:1789762156.283780:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762156.283783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762156.283787:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762156.283791:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762156.283794:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507602304:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789762156.283796:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762156.283808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507602816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762156.283815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762156.283818:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789762156.283820:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507602432:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762156.283821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762156.283829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2133 00000100:00080000:3.0:1789762156.283834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762156.283836:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762156.283840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762156.283848:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762156.283850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507602944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762156.283863:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffff8d25d1b08a80 x1876699507602432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762156.283875:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507602432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762156.283881:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762156.283884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762156.283886:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507602432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762156.283890:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507602560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762156.283902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762156.283909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffff8d25d1b0aa00 x1876699507602560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762156.283916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507602560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762156.283921:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762156.283923:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762156.283925:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507602560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762156.283928:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507602688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762156.283938:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762156.283944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffff8d25d1b09180 x1876699507602688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762156.283951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507602688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762156.283965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762156.283967:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762156.283969:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507602688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762156.283973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762156.283985:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762156.283991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffff8d25d1b0ad80 x1876699507602816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762156.283999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507602816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762156.284004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762156.284005:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762156.284007:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762156.284022:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762156.284033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762156.284039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762156/real 1789762156] req@ffff8d25d1b09c00 x1876699507602944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762156.284046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507602944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762181 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762156.284051:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762156.284053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762156.284055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507602944:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762156.347522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762157.371574:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762158.395574:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762159.419564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762160.443495:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762161.403452:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762161.403463:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762161.403468:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762161.403479:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2139 00000100:00080000:3.0:1789762161.403487:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762161.403494:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762161.403525:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507603072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762161.403548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762161.403550:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762161.403552:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762161.403555:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2139 00000100:00080000:3.0:1789762161.403557:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762161.403560:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762161.403567:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762161.403575:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507603200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762161.403582:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762161.403585:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762161.403586:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762161.403589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2139 00000100:00080000:3.0:1789762161.403592:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762161.403594:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762161.403599:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762161.403604:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507603072:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762161.403609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507603328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762161.403617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762161.403619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762161.403620:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762161.403623:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2139 00000100:00080000:3.0:1789762161.403626:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762161.403627:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762161.403631:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762161.403640:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507603456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762161.403646:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762161.403648:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762161.403650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762161.403652:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2139 00000100:00080000:3.0:1789762161.403656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762161.403658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762161.403661:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762161.403664:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762161.403673:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507603584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762161.403680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762161.403682:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762161.403683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762161.403686:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2139 00000100:00080000:3.0:1789762161.403688:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762161.403690:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762161.403693:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762161.403706:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762161/real 1789762161] req@ffff8d25d1b0b100 x1876699507603072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762161.403707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507603712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762161.403747:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507603072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762161.403756:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762161.403761:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762161.403765:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507603072:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762161.403774:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762161.403802:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762161.403819:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762161/real 1789762161] req@ffff8d25d1b0a300 x1876699507603200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762161.403829:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507603200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762161.403843:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762161.403848:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762161.403852:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762161.403877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762161.403897:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762161.403908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762161/real 1789762161] req@ffff8d25d1b08380 x1876699507603328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762161.403917:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507603328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762161.403928:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762161.403932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762161.403935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762161.403941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762161.403953:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762161.403961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762161/real 1789762161] req@ffff8d25d1b08000 x1876699507603456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762161.403973:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507603456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762161.403979:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762161.403982:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762161.403984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762161.403988:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507603584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762161.404002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762161.404011:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762161/real 1789762161] req@ffff8d25d1b09500 x1876699507603584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762161.404019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507603584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762161.404026:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762161.404030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762161.404034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507603584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762161.404041:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507603712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762161.404066:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762161.404086:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762161/real 1789762161] req@ffff8d25d1b0b480 x1876699507603712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762161.404096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507603712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762186 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762161.404102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762161.404105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762161.404107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507603712:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762161.467551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762162.491494:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762163.515519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762164.539563:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762165.563529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762166.523561:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762166.523575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762166.523582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762166.523597:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2144 00000100:00080000:0.0:1789762166.523612:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762166.523622:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762166.523688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507603840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762166.523737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762166.523743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762166.523745:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762166.523750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2144 00000100:00080000:0.0:1789762166.523762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762166.523767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762166.523777:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762166.523786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507603840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762166.523792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507603968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762166.523804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762166.523807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762166.523809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762166.523812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2144 00000100:00080000:0.0:1789762166.523815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762166.523818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762166.523822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762166.523831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507604096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762166.523839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762166.523841:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762166.523843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762166.523845:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2144 00000100:00080000:0.0:1789762166.523848:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762166.523853:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762166.523854:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762166.523862:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762166.523870:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507604224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762166.523880:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762166.523882:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762166.523884:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762166.523886:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2144 00000100:00080000:0.0:1789762166.523889:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762166.523891:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762166.523895:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762166.523904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507604352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762166.523913:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762166/real 1789762166] req@ffff8d25d25ba680 x1876699507603840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762166.523914:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762166.523923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762166.523927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762166.523936:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2144 00000100:00080000:0.0:1789762166.523941:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762166.523944:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762166.523948:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762166.523949:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507603840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762166.523958:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507604480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762166.523963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762166.523973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762166.523979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507603840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762166.524004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762166.524027:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762166.524039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762166/real 1789762166] req@ffff8d25d25bb800 x1876699507603968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762166.524049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507603968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762166.524055:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762166.524058:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762166.524060:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507603968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762166.524065:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762166.524077:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762166.524085:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762166/real 1789762166] req@ffff8d25d25b9f80 x1876699507604096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762166.524093:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507604096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762166.524100:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762166.524102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762166.524104:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762166.524108:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762166.524120:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762166.524127:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762166/real 1789762166] req@ffff8d25d25b8380 x1876699507604224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762166.524142:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507604224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762166.524148:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762166.524150:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762166.524152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762166.524155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507604352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762166.524167:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762166.524174:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762166/real 1789762166] req@ffff8d25d25b8700 x1876699507604352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762166.524182:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507604352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762166.524188:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762166.524190:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762166.524193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507604352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762166.524196:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507604480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762166.524208:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762166.524214:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762166/real 1789762166] req@ffff8d25d25bb100 x1876699507604480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762166.524222:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507604480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762191 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762166.524233:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762166.524235:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762166.524238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507604480:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762166.587671:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762167.611605:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762168.635584:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762169.659527:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762170.683560:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762171.643455:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762171.643466:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762171.643471:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762171.643482:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2149 00000100:00080000:0.0:1789762171.643497:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762171.643505:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762171.643547:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507604608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762171.643573:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762171.643576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762171.643582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762171.643585:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2149 00000100:00080000:0.0:1789762171.643588:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762171.643591:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762171.643600:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762171.643611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507604736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762171.643619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762171.643622:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762171.643623:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762171.643626:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2149 00000100:00080000:0.0:1789762171.643628:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762171.643631:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762171.643635:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762171.643647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507604864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762171.643671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762171.643676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762171.643678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762171.643684:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2149 00000100:00080000:0.0:1789762171.643688:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762171.643698:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762171.643699:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507604608:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762171.643709:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762171.643722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507604992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762171.643733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762171.643736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762171.643738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762171.643740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2149 00000100:00080000:0.0:1789762171.643744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762171.643746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762171.643750:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762171.643756:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507605120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762171.643765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762171.643768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762171.643769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762171.643771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2149 00000100:00080000:0.0:1789762171.643782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762171.643785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762171.643789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762171.643796:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762171.643801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507605248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762171.643851:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762171/real 1789762171] req@ffff8d25d25b9c00 x1876699507604608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762171.643874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507604608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762171.643885:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762171.643890:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762171.643894:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507604608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762171.643902:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762171.643931:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762171.643946:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762171/real 1789762171] req@ffff8d25d25b8e00 x1876699507604736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762171.643960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507604736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762171.643970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762171.643974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762171.643978:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762171.643986:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762171.644010:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762171.644023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762171/real 1789762171] req@ffff8d25d25bb480 x1876699507604864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762171.644036:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507604864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762171.644045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762171.644048:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762171.644051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762171.644081:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762171.644106:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762171.644120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762171/real 1789762171] req@ffff8d25d25bb100 x1876699507604992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762171.644133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507604992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762171.644143:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762171.644146:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762171.644167:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507604992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762171.644176:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507605120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762171.644206:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762171.644223:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762171/real 1789762171] req@ffff8d25d25b8700 x1876699507605120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762171.644238:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507605120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762171.644249:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762171.644254:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762171.644258:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507605120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762171.644268:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507605248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762171.644333:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762171.644347:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762171/real 1789762171] req@ffff8d25d25b8380 x1876699507605248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762171.644360:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507605248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762196 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762171.644369:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762171.644372:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762171.644376:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507605248:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762171.707521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762172.731582:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762173.755545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762174.779706:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762175.803524:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762176.763481:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762176.763492:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762176.763496:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762176.763507:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2154 00000100:00080000:1.0:1789762176.763516:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762176.763534:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762176.763580:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507605376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762176.763619:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762176.763623:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762176.763625:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762176.763630:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2154 00000100:00080000:1.0:1789762176.763634:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762176.763639:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762176.763648:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762176.763662:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507605504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762176.763685:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762176.763690:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762176.763693:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762176.763698:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2154 00000100:00100000:2.0:1789762176.763698:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507605376:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762176.763703:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762176.763711:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762176.763716:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762176.763727:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507605632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762176.763737:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762176.763741:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762176.763743:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762176.763747:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2154 00000100:00080000:1.0:1789762176.763751:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762176.763755:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762176.763760:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762176.763775:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507605760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762176.763786:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762176.763788:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762176.763793:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762176.763795:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762176.763800:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2154 00000100:00080000:1.0:1789762176.763803:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762176.763807:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762176.763813:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762176.763824:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507605888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762176.763831:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762176.763834:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762176.763836:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762176.763841:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2154 00000100:00100000:2.0:1789762176.763842:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762176/real 1789762176] req@ffff8d25c8168000 x1876699507605376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762176.763844:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762176.763859:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762176.763866:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762176.763878:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507606016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762176.763884:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507605376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762176.763897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762176.763901:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762176.763906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507605376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762176.763926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507605504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762176.763945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762176.763954:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762176/real 1789762176] req@ffff8d25c8169500 x1876699507605504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762176.763963:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507605504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762176.763969:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762176.763971:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762176.763973:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507605504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762176.763976:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507605632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762176.763987:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762176.763993:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762176/real 1789762176] req@ffff8d25c8168700 x1876699507605632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762176.764001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507605632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762176.764010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762176.764012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762176.764014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507605632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762176.764018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507605760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762176.764028:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762176.764033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762176/real 1789762176] req@ffff8d25c8168380 x1876699507605760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762176.764041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507605760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762176.764046:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762176.764048:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762176.764049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507605760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762176.764053:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507605888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762176.764063:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762176.764069:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762176/real 1789762176] req@ffff8d25c8168e00 x1876699507605888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762176.764076:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507605888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762176.764081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762176.764083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762176.764085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507605888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762176.764089:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507606016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762176.764099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762176.764105:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762176/real 1789762176] req@ffff8d25c8169f80 x1876699507606016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762176.764112:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507606016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762201 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762176.764117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762176.764118:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762176.764120:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507606016:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762176.827596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762177.851496:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762178.875546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762179.899502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762180.923492:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762181.883543:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762181.883556:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762181.883562:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762181.883575:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2159 00000100:00080000:3.0:1789762181.883587:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762181.883595:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762181.883641:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507606144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762181.883680:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762181.883685:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762181.883688:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762181.883693:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2159 00000100:00080000:3.0:1789762181.883698:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762181.883703:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762181.883715:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762181.883746:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507606272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762181.883762:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762181.883771:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507606144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762181.883772:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762181.883776:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762181.883787:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2159 00000100:00080000:3.0:1789762181.883793:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762181.883796:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762181.883804:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762181.883817:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507606400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762181.883830:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762181.883835:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762181.883838:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762181.883842:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2159 00000100:00080000:3.0:1789762181.883846:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762181.883851:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762181.883858:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762181.883871:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507606528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762181.883884:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762181.883894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762181.883896:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762181.883902:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762181.883910:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2159 00000100:00080000:3.0:1789762181.883917:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762181.883922:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762181.883928:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762181.883957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762181/real 1789762181] req@ffff8d25d1b0b480 x1876699507606144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762181.883963:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507606656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762181.884013:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762181.884024:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789762181.884025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507606144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762181.884033:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762181.884054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762181.884066:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762181.884071:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2159 00000100:00100000:2.0:1789762181.884080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507606144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762181.884083:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762181.884098:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762181.884107:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762181.884125:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507606272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762181.884131:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507606784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762181.884175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762181.884197:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762181/real 1789762181] req@ffff8d25d1b09500 x1876699507606272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762181.884244:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507606272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762181.884255:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762181.884259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762181.884264:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507606272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762181.884272:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507606400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762181.884300:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762181.884315:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762181/real 1789762181] req@ffff8d25d1b08000 x1876699507606400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762181.884329:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507606400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762181.884340:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762181.884343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762181.884347:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507606400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762181.884354:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507606528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762181.884373:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762181.884429:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762181/real 1789762181] req@ffff8d25d1b08380 x1876699507606528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762181.884452:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507606528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762181.884463:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762181.884467:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762181.884471:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507606528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762181.884478:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507606656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762181.884501:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762181.884513:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762181/real 1789762181] req@ffff8d25d1b0a300 x1876699507606656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762181.884527:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507606656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762181.884537:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762181.884540:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762181.884544:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507606656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762181.884571:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507606784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762181.884591:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762181.884603:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762181/real 1789762181] req@ffff8d25d1b0b100 x1876699507606784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762181.884617:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507606784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762206 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762181.884634:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762181.884638:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762181.884642:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507606784:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762181.947555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762182.971630:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762183.995545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762185.019499:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762186.043507:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762187.003453:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762187.003461:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762187.003465:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762187.003474:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2164 00000100:00080000:3.0:1789762187.003482:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762187.003489:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762187.003519:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507606912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762187.003543:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762187.003547:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762187.003549:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762187.003554:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2164 00000100:00080000:3.0:1789762187.003557:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762187.003562:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762187.003572:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762187.003587:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507607040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762187.003597:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762187.003599:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762187.003600:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762187.003604:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2164 00000100:00080000:3.0:1789762187.003606:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762187.003611:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762187.003613:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507606912:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789762187.003615:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762187.003626:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507607168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762187.003634:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762187.003637:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762187.003638:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762187.003641:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2164 00000100:00080000:3.0:1789762187.003643:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762187.003645:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762187.003648:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762187.003656:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507607296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762187.003662:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762187.003665:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762187.003666:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762187.003669:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2164 00000100:00080000:3.0:1789762187.003671:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762187.003674:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789762187.003680:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789762187.003680:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762187.003690:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507607424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762187.003697:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762187.003700:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762187.003701:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762187.003704:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2164 00000100:00080000:3.0:1789762187.003706:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762187.003708:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762187.003711:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762187.003718:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507607552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762187.003726:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffff8d25d1b09c00 x1876699507606912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762187.003743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507606912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762187.003751:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762187.003755:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762187.003757:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507606912:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762187.003763:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762187.003779:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762187.003788:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffff8d25d1b0ad80 x1876699507607040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762187.003796:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507607040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762187.003806:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762187.003808:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762187.003811:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762187.003831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762187.003841:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762187.003848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffff8d25d1b09180 x1876699507607168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762187.003855:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507607168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762187.003861:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762187.003863:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762187.003864:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762187.003868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762187.003879:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762187.003884:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffff8d25d1b0aa00 x1876699507607296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762187.003892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507607296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762187.003897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762187.003898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762187.003901:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762187.003905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507607424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762187.003914:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762187.003920:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffff8d25d1b08a80 x1876699507607424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762187.003927:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507607424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762187.003932:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762187.003934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762187.003935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507607424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762187.003938:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507607552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762187.003948:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762187.003954:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762187/real 1789762187] req@ffff8d25d1b0bb80 x1876699507607552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762187.003960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507607552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762212 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762187.003968:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762187.003971:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762187.003974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507607552:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762187.067482:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762188.091543:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762189.115526:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762190.139561:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762191.163503:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762192.123457:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762192.123467:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762192.123472:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762192.123483:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2169 00000100:00080000:3.0:1789762192.123491:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762192.123497:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762192.123529:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507607680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762192.123558:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762192.123563:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762192.123565:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762192.123570:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2169 00000100:00080000:3.0:1789762192.123574:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762192.123577:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762192.123585:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762192.123597:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507607808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762192.123605:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762192.123609:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507607680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762192.123611:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762192.123615:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762192.123622:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2169 00000100:00080000:3.0:1789762192.123626:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762192.123630:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762192.123636:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762192.123659:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507607936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762192.123667:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762192.123673:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762192.123673:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762192.123677:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762192.123682:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2169 00000100:00080000:3.0:1789762192.123686:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762192.123690:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762192.123694:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762192.123703:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507608064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762192.123711:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762192.123713:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762192.123714:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762192.123716:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2169 00000100:00080000:3.0:1789762192.123718:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762192.123720:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762192.123724:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762192.123733:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507608192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762192.123736:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffff8d25d1b09880 x1876699507607680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762192.123742:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762192.123750:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762192.123754:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762192.123761:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2169 00000100:00080000:3.0:1789762192.123764:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762192.123767:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762192.123770:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762192.123775:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507607680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.123784:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789762192.123784:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507608320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762192.123792:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762192.123799:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507607680:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762192.123828:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762192.123857:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762192.123872:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffff8d25d1b08e00 x1876699507607808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.123881:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507607808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.123887:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.123889:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762192.123892:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762192.123896:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762192.123910:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762192.123917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffff8d25d1b0b800 x1876699507607936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.123925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507607936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.123930:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.123932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762192.123933:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507607936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762192.123937:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762192.123947:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762192.123953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffff8d25d1b08700 x1876699507608064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.123961:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507608064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.123970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.123973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762192.123977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762192.123983:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507608192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762192.124002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762192.124014:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffff8d25d1b0a680 x1876699507608192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.124025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507608192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.124034:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.124036:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762192.124038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507608192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762192.124042:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507608320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762192.124056:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762192.124063:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762192/real 1789762192] req@ffff8d25d1b0bb80 x1876699507608320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762192.124070:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507608320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762217 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762192.124075:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762192.124077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762192.124079:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507608320:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762192.187532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762193.211530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762194.235546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762195.259525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762196.283519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762197.243519:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.243530:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762197.243535:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.243546:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2174 00000100:00080000:0.0:1789762197.243558:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.243567:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762197.243611:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507608448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762197.243640:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.243643:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762197.243645:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.243648:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2174 00000100:00080000:0.0:1789762197.243651:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.243655:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762197.243665:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762197.243675:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507608448:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762197.243679:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507608576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762197.243691:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.243695:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762197.243697:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.243703:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2174 00000100:00080000:0.0:1789762197.243708:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.243713:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762197.243718:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762197.243730:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507608704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762197.243740:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.243743:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762197.243745:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.243748:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2174 00000100:00080000:0.0:1789762197.243751:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.243753:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762197.243758:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762197.243770:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507608832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762197.243778:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.243781:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762197.243782:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.243785:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2174 00000100:00080000:0.0:1789762197.243788:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.243790:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762197.243796:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762197.243809:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507608960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762197.243816:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762197.243819:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762197.243822:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762197.243825:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2174 00000100:00080000:0.0:1789762197.243830:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762197.243833:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762197.243839:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762197.243851:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507609088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762197.244048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762197.244078:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762197.244114:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762197.244135:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507608960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762197.244155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507609088:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762197.307540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762198.331581:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762199.355508:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789762200.315632:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1789762200.315668:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762200.315680:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762200.315725:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762200.315732:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762200.315739:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762200.315745:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762200.315753:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789762200.315803:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762200] req@ffff8d25d25bb800 x1876699507608576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762200.315855:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507608576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762200.315872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762200.315878:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762200.315884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762200.315894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762200] req@ffff8d25d25ba680 x1876699507608704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762200.315915:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507608704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762200.315933:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762200.315938:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762200.315941:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762200.315950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762200] req@ffff8d25d25bad80 x1876699507608832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762200.315964:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507608832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762200.315973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762200.315977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762200.315980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507608832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762200.315988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762200] req@ffff8d25d25bbb80 x1876699507608960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762200.316003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507608960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762200.316010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762200.316013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762200.316015:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507608960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762200.316020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762200] req@ffff8d25d25b9180 x1876699507609088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762200.316028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507609088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762200.316037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762200.316042:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762200.316046:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507609088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762200.316054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762197/real 1789762200] req@ffff8d25d25b9f80 x1876699507608448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762200.316069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507608448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762222 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762200.316097:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762200.316102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762200.316105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507608448:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1789762200.379538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762201.403491:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762202.363483:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762202.363498:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762202.363504:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762202.363519:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2180 00000100:00080000:3.0:1789762202.363531:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762202.363541:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762202.363589:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507609216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762202.363631:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762202.363636:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762202.363639:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762202.363645:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2180 00000100:00080000:3.0:1789762202.363650:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762202.363655:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762202.363668:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762202.363683:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507609344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762202.363696:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762202.363705:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789762202.363707:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507609216:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762202.363708:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762202.363722:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2180 00000100:00080000:3.0:1789762202.363728:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762202.363734:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762202.363741:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762202.363758:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507609472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762202.363770:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762202.363774:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762202.363777:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762202.363782:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2180 00000100:00080000:3.0:1789762202.363789:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762202.363795:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762202.363802:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789762202.363809:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762202.363827:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507609600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762202.363845:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762202.363849:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762202.363851:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762202.363857:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2180 00000100:00080000:3.0:1789762202.363863:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762202.363869:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762202.363875:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762202.363891:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507609728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762202.363904:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffff8d25d1b08a80 x1876699507609216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762202.363908:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762202.363925:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762202.363936:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762202.363954:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2180 00000100:00080000:3.0:1789762202.363962:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762202.363970:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507609216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762202.363975:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762202.363990:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789762202.363998:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789762202.364012:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507609856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762202.364018:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762202.364035:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507609216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762202.364097:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507609344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762202.364130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762202.364145:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffff8d25d1b0aa00 x1876699507609344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762202.364159:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507609344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762202.364168:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762202.364172:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762202.364175:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507609344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762202.364183:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507609472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762202.364207:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762202.364220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffff8d25d1b09180 x1876699507609472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762202.364233:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507609472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762202.364242:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762202.364246:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762202.364250:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507609472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762202.364256:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507609600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762202.364278:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762202.364291:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffff8d25d1b0ad80 x1876699507609600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762202.364304:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507609600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762202.364312:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762202.364316:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762202.364319:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507609600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762202.364326:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507609728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762202.364347:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762202.364359:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffff8d25d1b09c00 x1876699507609728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762202.364407:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507609728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762202.364416:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762202.364420:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762202.364423:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507609728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762202.364440:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507609856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762202.364463:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762202.364479:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762202/real 1789762202] req@ffff8d25d1b0b100 x1876699507609856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762202.364492:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507609856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762227 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762202.364500:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762202.364504:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762202.364507:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507609856:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762202.427577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762203.451528:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762204.475586:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762205.499568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762206.523494:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762207.483487:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762207.483496:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762207.483500:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762207.483508:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2185 00000100:00080000:0.0:1789762207.483516:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762207.483522:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762207.483554:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507609984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762207.483576:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762207.483579:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762207.483580:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762207.483583:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2185 00000100:00080000:0.0:1789762207.483586:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762207.483593:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762207.483600:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762207.483610:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507610112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762207.483617:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762207.483619:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762207.483620:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762207.483622:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2185 00000100:00080000:0.0:1789762207.483625:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762207.483628:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762207.483632:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762207.483640:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507610240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762207.483647:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762207.483649:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762207.483650:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762207.483652:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2185 00000100:00080000:0.0:1789762207.483654:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762207.483656:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762207.483661:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762207.483667:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507610368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762207.483681:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507609984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762207.483692:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762207.483698:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762207.483700:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762207.483707:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2185 00000100:00080000:0.0:1789762207.483712:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762207.483718:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762207.483724:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762207.483742:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507610496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762207.483749:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762207.483762:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762207.483764:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762207.483766:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2185 00000100:00080000:0.0:1789762207.483777:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762207.483780:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762207.483785:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762207.483791:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762207.483796:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507610624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762207.483865:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762207/real 1789762207] req@ffff8d25d25b8a80 x1876699507609984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762207.483888:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507609984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762207.483913:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762207.483917:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762207.483920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507609984:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762207.483928:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762207.483948:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762207.483958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762207/real 1789762207] req@ffff8d25d25ba300 x1876699507610112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762207.483983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507610112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762207.483989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762207.483991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762207.483993:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762207.483997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762207.484009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762207.484016:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762207/real 1789762207] req@ffff8d25d25b8000 x1876699507610240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762207.484038:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507610240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762207.484048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762207.484051:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762207.484053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762207.484079:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762207.484095:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762207.484102:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762207/real 1789762207] req@ffff8d25d25b9f80 x1876699507610368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762207.484110:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507610368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762207.484116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762207.484119:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762207.484121:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762207.484124:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507610496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762207.484135:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762207.484141:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762207/real 1789762207] req@ffff8d25d25b9180 x1876699507610496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762207.484148:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507610496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762207.484154:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762207.484156:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762207.484158:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507610496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762207.484162:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507610624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762207.484172:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762207.484178:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762207/real 1789762207] req@ffff8d25d25bbb80 x1876699507610624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762207.484184:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507610624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762232 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762207.484189:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762207.484191:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762207.484193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507610624:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762207.547581:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762208.571548:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762209.595611:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762210.619521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762211.643569:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762212.603474:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762212.603485:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762212.603491:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762212.603505:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2190 00000100:00080000:0.0:1789762212.603518:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762212.603526:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762212.603598:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507610752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762212.603646:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762212.603654:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762212.603656:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762212.603662:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2190 00000100:00080000:0.0:1789762212.603666:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762212.603672:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762212.603683:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762212.603696:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507610880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762212.603703:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507610752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762212.603707:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762212.603711:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762212.603713:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762212.603717:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2190 00000100:00080000:0.0:1789762212.603719:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762212.603722:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762212.603726:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762212.603735:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507611008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762212.603742:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762212.603745:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762212.603747:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762212.603750:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2190 00000100:00080000:0.0:1789762212.603754:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762212.603757:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762212.603761:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762212.603768:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507611136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762212.603776:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762212.603781:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:2.0:1789762212.603784:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762212.603786:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762212.603793:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2190 00000100:00080000:0.0:1789762212.603798:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762212.603803:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762212.603809:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762212.603822:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507611264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762212.603833:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762212/real 1789762212] req@ffff8d25d25bad80 x1876699507610752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762212.603835:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762212.603844:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762212.603847:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762212.603852:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2190 00000100:00080000:0.0:1789762212.603856:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762212.603860:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507610752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762212.603863:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762212.603872:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789762212.603877:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762212.603883:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762212.603885:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507611392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762212.603890:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507610752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762212.603924:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762212.603941:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762212.603950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762212/real 1789762212] req@ffff8d25d25ba680 x1876699507610880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762212.603970:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507610880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762212.603976:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762212.603978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762212.603997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507610880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762212.604002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762212.604018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762212.604037:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762212/real 1789762212] req@ffff8d25d25bb800 x1876699507611008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762212.604045:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507611008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762212.604054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762212.604057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762212.604061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762212.604067:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762212.604090:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762212.604102:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762212/real 1789762212] req@ffff8d25d25b8380 x1876699507611136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762212.604112:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507611136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762212.604118:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762212.604120:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762212.604123:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762212.604127:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507611264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762212.604142:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762212.604149:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762212/real 1789762212] req@ffff8d25d25b8700 x1876699507611264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762212.604157:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507611264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762212.604162:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762212.604163:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762212.604165:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507611264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762212.604169:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507611392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762212.604180:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762212.604185:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762212/real 1789762212] req@ffff8d25d25bb100 x1876699507611392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762212.604193:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507611392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762237 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762212.604203:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762212.604205:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762212.604207:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507611392:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762212.667511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762213.691562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762214.715697:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762215.739609:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762216.763584:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762217.723488:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762217.723497:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762217.723500:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762217.723510:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2195 00000100:00080000:3.0:1789762217.723519:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762217.723527:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762217.723558:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507611520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762217.723580:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762217.723583:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762217.723584:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762217.723588:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2195 00000100:00080000:3.0:1789762217.723590:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762217.723592:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762217.723600:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762217.723609:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507611648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762217.723616:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762217.723618:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762217.723620:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762217.723622:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2195 00000100:00080000:3.0:1789762217.723624:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762217.723630:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507611520:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762217.723630:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762217.723634:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762217.723644:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507611776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762217.723652:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762217.723654:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762217.723655:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762217.723657:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2195 00000100:00080000:3.0:1789762217.723660:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762217.723663:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762217.723666:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762217.723673:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507611904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762217.723680:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762217.723683:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762217.723684:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762217.723686:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2195 00000100:00080000:3.0:1789762217.723689:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762217.723693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762217.723697:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762217.723701:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762217.723710:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507612032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762217.723716:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762217.723719:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762217.723720:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762217.723724:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2195 00000100:00080000:3.0:1789762217.723727:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762217.723730:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762217.723734:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762217/real 1789762217] req@ffff8d25d1b0a300 x1876699507611520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762217.723734:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762217.723748:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507612160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762217.723756:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507611520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762217.723765:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762217.723769:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762217.723773:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507611520:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762217.723779:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762217.723794:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762217.723802:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762217/real 1789762217] req@ffff8d25d1b08380 x1876699507611648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762217.723810:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507611648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762217.723820:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762217.723823:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762217.723825:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762217.723846:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762217.723856:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762217.723863:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762217/real 1789762217] req@ffff8d25d1b08000 x1876699507611776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762217.723871:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507611776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762217.723876:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762217.723878:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762217.723880:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762217.723884:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762217.723894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762217.723900:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762217/real 1789762217] req@ffff8d25d1b09500 x1876699507611904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762217.723907:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507611904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762217.723912:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762217.723913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762217.723916:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507611904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762217.723919:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762217.723929:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762217.723934:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762217/real 1789762217] req@ffff8d25d1b0b480 x1876699507612032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762217.723955:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507612032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762217.723960:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762217.723963:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762217.723965:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762217.723969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762217.723984:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762217.723992:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762217/real 1789762217] req@ffff8d25d1b0b100 x1876699507612160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762217.724000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507612160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762242 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762217.724005:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762217.724007:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762217.724008:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612160:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762217.787599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762218.811556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762219.835547:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762220.859506:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762221.883489:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762222.843465:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762222.843473:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762222.843477:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762222.843486:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2200 00000100:00080000:3.0:1789762222.843494:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762222.843500:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762222.843532:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507612288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762222.843559:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762222.843563:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762222.843565:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762222.843570:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2200 00000100:00080000:3.0:1789762222.843574:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762222.843578:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762222.843588:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762222.843603:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507612416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762222.843613:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762222.843616:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507612288:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762222.843616:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762222.843620:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762222.843627:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2200 00000100:00080000:3.0:1789762222.843630:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762222.843633:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762222.843637:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762222.843646:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507612544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762222.843653:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762222.843655:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762222.843657:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762222.843660:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2200 00000100:00080000:3.0:1789762222.843671:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762222.843673:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762222.843677:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789762222.843681:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762222.843689:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507612672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762222.843697:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762222.843699:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762222.843701:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762222.843703:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2200 00000100:00080000:3.0:1789762222.843706:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762222.843708:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762222.843712:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762222.843720:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507612800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762222.843727:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762222/real 1789762222] req@ffff8d25d1b09c00 x1876699507612288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762222.843728:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762222.843734:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762222.843737:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762222.843744:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2200 00000100:00080000:3.0:1789762222.843749:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762222.843753:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762222.843756:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762222.843764:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507612288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762222.843765:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507612928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762222.843796:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762222.843801:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762222.843804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507612288:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762222.843826:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507612416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762222.843846:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762222.843855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762222/real 1789762222] req@ffff8d25d1b0ad80 x1876699507612416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762222.843864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507612416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762222.843870:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762222.843872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762222.843874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507612416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762222.843878:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507612544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762222.843889:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762222.843896:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762222/real 1789762222] req@ffff8d25d1b09180 x1876699507612544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762222.843925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507612544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762222.843932:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762222.843934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762222.843936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507612544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762222.843940:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507612672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762222.843956:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762222.843964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762222/real 1789762222] req@ffff8d25d1b0aa00 x1876699507612672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762222.843971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507612672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762222.843977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762222.843979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762222.843981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507612672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762222.843984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762222.843998:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762222.844009:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762222/real 1789762222] req@ffff8d25d1b08a80 x1876699507612800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762222.844021:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507612800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762222.844037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762222.844041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762222.844045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762222.844049:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762222.844065:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762222.844072:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762222/real 1789762222] req@ffff8d25d1b0bb80 x1876699507612928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762222.844080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507612928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762247 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762222.844085:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762222.844087:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762222.844089:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507612928:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762222.907575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762223.931527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762224.955558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762225.979556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762227.003559:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762227.963499:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762227.963508:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762227.963511:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762227.963520:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2205 00000100:00080000:0.0:1789762227.963529:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762227.963537:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762227.963575:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507613056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762227.963604:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762227.963607:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762227.963608:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762227.963611:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2205 00000100:00080000:0.0:1789762227.963614:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762227.963618:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762227.963625:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762227.963638:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507613184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762227.963647:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762227.963649:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762227.963650:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762227.963653:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2205 00000100:00080000:0.0:1789762227.963656:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762227.963660:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762227.963667:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762227.963669:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507613056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762227.963677:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507613312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762227.963693:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762227.963696:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762227.963697:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762227.963699:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2205 00000100:00080000:0.0:1789762227.963701:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762227.963702:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762227.963706:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762227.963716:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507613440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762227.963722:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762227.963724:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762227.963725:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762227.963728:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2205 00000100:00080000:0.0:1789762227.963730:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762227.963733:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762227.963736:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762227.963749:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507613568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762227.963763:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762227.963765:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762227.963767:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762227.963769:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2205 00000100:00080000:0.0:1789762227.963771:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762227.963773:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762227.963776:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762227.963783:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762227.963785:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507613696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762227.963851:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762227/real 1789762227] req@ffff8d25d25bb480 x1876699507613056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762227.963885:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507613056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762227.963900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762227.963904:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762227.963907:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507613056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762227.963939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762227.963970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762227.963986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762227/real 1789762227] req@ffff8d25d25b8e00 x1876699507613184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762227.964001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507613184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762227.964011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762227.964014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762227.964018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762227.964026:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762227.964050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762227.964061:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762227/real 1789762227] req@ffff8d25d25b9c00 x1876699507613312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762227.964070:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507613312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762227.964076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762227.964079:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762227.964081:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762227.964086:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762227.964102:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762227.964109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762227/real 1789762227] req@ffff8d25d25bb100 x1876699507613440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762227.964117:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507613440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762227.964123:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762227.964125:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762227.964126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762227.964130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507613568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762227.964146:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762227.964157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762227/real 1789762227] req@ffff8d25d25b8700 x1876699507613568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762227.964169:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507613568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762227.964179:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762227.964183:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762227.964187:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507613568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762227.964194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507613696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762227.964218:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762227.964231:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762227/real 1789762227] req@ffff8d25d25b8380 x1876699507613696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762227.964245:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507613696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762252 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762227.964252:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762227.964254:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762227.964257:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507613696:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762228.027536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762229.051526:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762230.075504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762231.099570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762232.123552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762233.083501:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.083512:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762233.083517:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.083525:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2210 00000100:00080000:2.0:1789762233.083546:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.083554:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762233.083585:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507613824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762233.083610:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.083614:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762233.083615:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.083619:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2210 00000100:00080000:2.0:1789762233.083624:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.083629:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762233.083640:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762233.083656:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507613824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762233.083663:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507613952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762233.083678:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.083681:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762233.083684:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.083688:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2210 00000100:00080000:2.0:1789762233.083706:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.083710:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762233.083717:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762233.083717:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762233.083733:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507614080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762233.083744:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.083747:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762233.083750:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.083755:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2210 00000100:00080000:2.0:1789762233.083760:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762233.083763:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffff8d25c8020700 x1876699507613824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762233.083768:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762233.083779:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762233.083794:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507614208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762233.083805:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.083808:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762233.083810:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.083813:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2210 00000100:00080000:2.0:1789762233.083815:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.083818:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762233.083822:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762233.083845:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507614336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762233.083852:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762233.083854:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762233.083856:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762233.083858:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2210 00000100:00080000:2.0:1789762233.083861:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762233.083863:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762233.083866:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762233.083878:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507613824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762233.083880:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507614464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762233.084003:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762233.084007:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762233.084011:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507613824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762233.084071:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762233.084095:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762233.084107:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffff8d25c8023b80 x1876699507613952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762233.084116:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507613952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762233.084122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762233.084124:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762233.084126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507613952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762233.084130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762233.084141:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762233.084147:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffff8d25c8023100 x1876699507614080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762233.084155:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507614080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762233.084160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762233.084162:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762233.084164:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762233.084168:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762233.084179:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762233.084185:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffff8d25c8022d80 x1876699507614208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762233.084192:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507614208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762233.084197:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762233.084199:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762233.084201:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762233.084204:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507614336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762233.084214:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762233.084220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffff8d25c8020380 x1876699507614336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762233.084227:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507614336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762233.084232:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762233.084234:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762233.084236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507614336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762233.084239:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507614464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762233.084248:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762233.084254:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762233/real 1789762233] req@ffff8d25c8022300 x1876699507614464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762233.084263:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507614464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762258 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762233.084268:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762233.084269:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762233.084271:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507614464:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762233.147539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762234.171506:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762235.195645:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762236.219550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762237.243524:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762238.203501:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762238.203512:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762238.203517:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762238.203527:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2215 00000100:00080000:0.0:1789762238.203538:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762238.203545:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762238.203586:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507614592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762238.203630:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762238.203634:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762238.203637:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762238.203643:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2215 00000100:00080000:0.0:1789762238.203648:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762238.203654:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762238.203668:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762238.203678:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507614592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762238.203683:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507614720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762238.203695:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762238.203709:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762238.203711:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762238.203716:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2215 00000100:00080000:0.0:1789762238.203720:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762238.203724:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762238.203729:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762238.203757:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507614848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762238.203765:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762238.203768:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762238.203769:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762238.203773:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2215 00000100:00080000:0.0:1789762238.203776:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762238.203777:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762238.203779:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762238.203784:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762238.203810:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507614976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762238.203820:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762238.203823:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762238.203826:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762238.203846:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2215 00000100:00080000:0.0:1789762238.203850:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762238.203853:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffff8d25d25bb800 x1876699507614592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762238.203854:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762238.203860:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762238.203871:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507615104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762238.203875:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507614592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762238.203883:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762238.203887:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762238.203889:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762238.203890:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762238.203893:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2215 00000100:00080000:3.0:1789762238.203894:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762238.203897:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762238.203899:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507614592:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762238.203901:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762238.203906:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762238.203917:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507615232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762238.203930:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762238.203957:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762238.203976:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffff8d25d25ba680 x1876699507614720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762238.203987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507614720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762238.203995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762238.203997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762238.204000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762238.204005:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762238.204023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762238.204032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffff8d25d25bad80 x1876699507614848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762238.204040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507614848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762238.204062:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762238.204064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762238.204066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762238.204069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762238.204082:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762238.204088:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffff8d25d25bbb80 x1876699507614976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762238.204096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507614976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762238.204101:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762238.204103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762238.204105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507614976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762238.204109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507615104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762238.204120:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762238.204126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffff8d25d25b9180 x1876699507615104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762238.204133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507615104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762238.204138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762238.204140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762238.204142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507615104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762238.204145:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507615232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762238.204155:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762238.204160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762238/real 1789762238] req@ffff8d25d25b9f80 x1876699507615232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762238.204167:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507615232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762263 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762238.204172:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762238.204175:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762238.204177:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507615232:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762238.267569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762239.291514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762240.315587:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762241.339584:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762242.363566:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762243.323519:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762243.323530:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762243.323534:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762243.323545:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2221 00000100:00080000:2.0:1789762243.323556:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762243.323564:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762243.323595:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507615360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762243.323629:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762243.323632:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762243.323637:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762243.323640:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2221 00000100:00080000:2.0:1789762243.323643:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762243.323646:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762243.323655:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762243.323664:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507615488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762243.323672:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507615360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762243.323676:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762243.323680:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762243.323681:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762243.323684:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2221 00000100:00080000:2.0:1789762243.323687:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762243.323690:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762243.323694:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762243.323703:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507615616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762243.323714:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762243.323717:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762243.323719:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762243.323721:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2221 00000100:00080000:2.0:1789762243.323723:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762243.323726:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762243.323732:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762243.323739:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762243.323765:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507615744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762243.323774:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762243.323776:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762243.323778:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762243.323781:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2221 00000100:00080000:2.0:1789762243.323784:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762243.323786:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffff8d25c8020000 x1876699507615360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762243.323802:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762243.323810:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762243.323821:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507615360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762243.323823:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507615872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762243.323852:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762243.323852:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762243.323870:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762243.323872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762243.323873:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762243.323882:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2221 00000100:00100000:3.0:1789762243.323882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507615360:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762243.323888:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762243.323892:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762243.323895:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762243.323902:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507616000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762243.323909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507615488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762243.323931:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762243.323942:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffff8d25c8023800 x1876699507615488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.323951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507615488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.323957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.323959:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762243.323961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507615488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762243.323965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507615616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762243.323976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762243.323983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffff8d25c8022a00 x1876699507615616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.323995:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507615616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.324000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.324002:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762243.324004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507615616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762243.324013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507615744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762243.324026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762243.324037:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffff8d25c8022680 x1876699507615744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.324049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507615744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.324058:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.324061:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762243.324065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507615744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762243.324071:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507615872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762243.324091:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762243.324099:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffff8d25c8021180 x1876699507615872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.324107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507615872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.324128:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.324130:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762243.324132:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507615872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762243.324144:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507616000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762243.324156:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762243.324162:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762243/real 1789762243] req@ffff8d25c8020a80 x1876699507616000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762243.324170:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507616000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762268 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762243.324175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762243.324176:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762243.324178:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507616000:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762243.387505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762244.411683:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762245.435527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762246.459610:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762247.483564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762248.443513:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762248.443525:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762248.443529:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762248.443540:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2226 00000100:00080000:0.0:1789762248.443550:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762248.443557:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762248.443607:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507616128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762248.443647:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762248.443652:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762248.443654:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762248.443659:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2226 00000100:00080000:0.0:1789762248.443663:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762248.443668:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762248.443677:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762248.443688:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507616128:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762248.443692:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507616256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762248.443704:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762248.443706:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762248.443708:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762248.443726:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2226 00000100:00080000:0.0:1789762248.443730:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762248.443735:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762248.443741:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762248.443755:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507616384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762248.443766:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762248.443769:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762248.443771:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762248.443774:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2226 00000100:00080000:0.0:1789762248.443779:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762248.443782:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762248.443787:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762248.443793:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762248.443801:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507616512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762248.443810:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762248.443813:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762248.443815:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762248.443817:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2226 00000100:00080000:0.0:1789762248.443828:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762248.443831:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762248.443834:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762248.443841:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507616640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762248.443847:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762248.443850:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762248.443852:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762248.443855:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2226 00000100:00080000:0.0:1789762248.443857:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762248.443859:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762248.443863:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762248.443865:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762248/real 1789762248] req@ffff8d25d25b8000 x1876699507616128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762248.443872:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507616768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762248.443911:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507616128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762248.443935:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762248.443942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762248.443947:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507616128:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762248.443976:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507616256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762248.444004:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762248.444020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762248/real 1789762248] req@ffff8d25d25ba300 x1876699507616256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762248.444041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507616256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762248.444051:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762248.444055:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762248.444059:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507616256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762248.444070:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507616384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762248.444090:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762248.444099:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762248/real 1789762248] req@ffff8d25d25b8a80 x1876699507616384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762248.444107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507616384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762248.444113:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762248.444115:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762248.444117:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507616384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762248.444121:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507616512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762248.444134:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762248.444141:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762248/real 1789762248] req@ffff8d25d25b9f80 x1876699507616512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762248.444148:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507616512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762248.444154:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762248.444155:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762248.444157:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507616512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762248.444160:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507616640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762248.444171:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762248.444177:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762248/real 1789762248] req@ffff8d25d25b9180 x1876699507616640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762248.444184:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507616640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762248.444190:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762248.444192:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762248.444193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507616640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762248.444196:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507616768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762248.444207:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762248.444213:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762248/real 1789762248] req@ffff8d25d25bbb80 x1876699507616768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762248.444225:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507616768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762273 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762248.444230:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762248.444232:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762248.444234:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507616768:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762248.507616:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762249.531511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762250.555582:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762251.579658:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762252.603598:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762253.563492:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762253.563504:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762253.563509:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762253.563521:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2231 00000100:00080000:0.0:1789762253.563532:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762253.563540:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762253.563582:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507616896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762253.563612:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762253.563617:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762253.563620:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762253.563625:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2231 00000100:00080000:0.0:1789762253.563630:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762253.563636:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762253.563648:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762253.563665:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507617024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762253.563693:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507616896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762253.563696:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762253.563700:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762253.563702:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762253.563707:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2231 00000100:00080000:0.0:1789762253.563711:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762253.563716:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762253.563723:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762253.563735:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507617152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762253.563746:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762253.563750:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762253.563752:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762253.563756:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2231 00000100:00080000:0.0:1789762253.563760:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762253.563764:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762253.563764:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762253.563770:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762253.563782:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507617280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762253.563794:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762253.563797:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762253.563799:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762253.563804:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2231 00000100:00080000:0.0:1789762253.563808:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762253.563811:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762253/real 1789762253] req@ffff8d25d25bad80 x1876699507616896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762253.563812:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762253.563819:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762253.563830:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507617408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762253.563833:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507616896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762253.563843:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762253.563847:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762253.563848:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762253.563849:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762253.563853:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2231 00000100:00080000:3.0:1789762253.563853:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762253.563857:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762253.563858:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507616896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762253.563862:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762253.563867:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762253.563878:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507617536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762253.563886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762253.563916:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762253.563933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762253/real 1789762253] req@ffff8d25d25ba680 x1876699507617024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762253.563967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507617024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762253.563981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762253.563986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762253.563991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762253.564000:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762253.564024:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762253.564036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762253/real 1789762253] req@ffff8d25d25bb800 x1876699507617152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762253.564047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507617152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762253.564069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762253.564073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762253.564076:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762253.564082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762253.564117:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762253.564126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762253/real 1789762253] req@ffff8d25d25b8380 x1876699507617280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762253.564136:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507617280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762253.564143:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762253.564146:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762253.564149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762253.564154:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507617408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762253.564167:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762253.564175:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762253/real 1789762253] req@ffff8d25d25b8700 x1876699507617408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762253.564184:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507617408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762253.564190:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762253.564192:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762253.564195:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507617408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762253.564210:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507617536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762253.564223:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762253.564231:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762253/real 1789762253] req@ffff8d25d25bb100 x1876699507617536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762253.564241:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507617536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762278 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762253.564247:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762253.564249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762253.564251:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507617536:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762253.627512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762254.651582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762255.675575:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762256.699522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762257.723540:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762258.683476:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762258.683487:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762258.683491:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762258.683503:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2236 00000100:00080000:0.0:1789762258.683513:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762258.683521:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762258.683586:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507617664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762258.683615:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762258.683618:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762258.683620:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762258.683624:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2236 00000100:00080000:0.0:1789762258.683626:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762258.683629:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762258.683653:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762258.683678:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507617792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762258.683696:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762258.683700:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762258.683703:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762258.683708:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2236 00000100:00080000:0.0:1789762258.683714:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762258.683721:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762258.683729:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507617664:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762258.683730:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762258.683748:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507617920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762258.683760:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762258.683764:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762258.683766:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762258.683772:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2236 00000100:00080000:0.0:1789762258.683777:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762258.683783:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762258.683789:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762258.683805:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762258.683811:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507618048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762258.683825:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762258.683828:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762258.683830:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762258.683835:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2236 00000100:00080000:0.0:1789762258.683840:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762258.683844:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762258.683850:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762258.683857:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762258/real 1789762258] req@ffff8d25d25b9c00 x1876699507617664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762258.683860:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507618176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762258.683897:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762258.683918:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762258.683921:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762258.683926:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2236 00000100:00080000:0.0:1789762258.683929:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762258.683935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507617664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762258.683942:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762258.683954:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762258.683961:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762258.683966:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507618304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762258.683967:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762258.683983:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507617664:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762258.683989:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762258.684030:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762258.684042:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762258/real 1789762258] req@ffff8d25d25b8e00 x1876699507617792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762258.684050:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507617792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762258.684056:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762258.684059:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762258.684061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762258.684080:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762258.684091:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762258.684098:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762258/real 1789762258] req@ffff8d25d25bb480 x1876699507617920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762258.684109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507617920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762258.684115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762258.684116:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762258.684118:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507617920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762258.684122:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762258.684133:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762258.684138:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762258/real 1789762258] req@ffff8d25d25bb100 x1876699507618048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762258.684146:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507618048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762258.684160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762258.684162:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762258.684164:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762258.684168:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507618176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762258.684180:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762258.684198:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762258/real 1789762258] req@ffff8d25d25b8700 x1876699507618176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762258.684205:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507618176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762258.684214:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762258.684217:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762258.684219:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507618176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762258.684225:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507618304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762258.684245:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762258.684256:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762258/real 1789762258] req@ffff8d25d25b8380 x1876699507618304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762258.684269:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507618304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762283 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762258.684278:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762258.684281:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762258.684283:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507618304:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762258.747723:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762259.771508:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762260.795517:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762261.819509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762262.843543:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762263.803468:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762263.803477:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762263.803480:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762263.803489:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2241 00000100:00080000:0.0:1789762263.803497:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762263.803503:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762263.803533:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507618432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762263.803557:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762263.803560:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762263.803562:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762263.803565:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2241 00000100:00080000:0.0:1789762263.803567:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762263.803571:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762263.803580:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762263.803594:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507618560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762263.803616:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762263.803619:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762263.803622:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762263.803625:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2241 00000100:00080000:0.0:1789762263.803628:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762263.803632:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762263.803636:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762263.803646:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507618688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762263.803655:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762263.803655:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507618432:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762263.803661:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762263.803664:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762263.803669:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2241 00000100:00080000:0.0:1789762263.803671:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762263.803673:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762263.803677:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762263.803686:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507618816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762263.803692:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762263.803694:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762263.803696:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762263.803698:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2241 00000100:00080000:0.0:1789762263.803700:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762263.803702:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762263.803705:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762263.803712:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507618944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762263.803719:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762263.803721:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762263.803722:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762263.803724:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2241 00000100:00080000:0.0:1789762263.803726:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762263.803729:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762263.803732:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762263.803738:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507619072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762263.803769:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762263.803804:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762263.803821:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762263.803845:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507618944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762263.803863:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507619072:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762263.867572:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762264.891519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762265.915496:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762266.875677:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1789762266.875731:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762266.875748:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762266.875794:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762266.875805:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762266.875814:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762266.875827:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789762266.875842:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762263/real 1789762266] req@ffff8d25d25bad80 x1876699507618688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:3.0:1789762266.875843:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789762266.875868:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507618688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762266.875878:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762266.875881:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762266.875884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762266.875891:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762263/real 1789762266] req@ffff8d25d25bbb80 x1876699507618816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762266.875899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507618816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762266.875904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762266.875906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762266.875908:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762266.875912:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762263/real 1789762266] req@ffff8d25d25b9180 x1876699507618944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762266.875920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507618944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762266.875925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762266.875927:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762266.875929:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507618944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762266.875933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762263/real 1789762266] req@ffff8d25d25b9f80 x1876699507619072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762266.875940:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507619072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762266.875944:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762266.875946:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762266.875948:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507619072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762266.875953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762263/real 1789762266] req@ffff8d25d25bb800 x1876699507618432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762266.875966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507618432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762266.875974:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762266.875977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762266.875981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507618432:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762266.875988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762263/real 1789762266] req@ffff8d25d25ba680 x1876699507618560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762266.876000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507618560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762288 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762266.876008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762266.876012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762266.876015:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507618560:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762266.939486:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762267.963514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762268.923580:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762268.923606:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762268.923611:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762268.923621:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2246 00000100:00080000:2.0:1789762268.923631:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762268.923639:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762268.923674:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507619200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762268.923701:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762268.923704:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762268.923706:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762268.923709:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2246 00000100:00080000:2.0:1789762268.923712:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762268.923716:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762268.923725:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762268.923735:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507619328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762268.923743:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762268.923748:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507619200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762268.923749:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762268.923754:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762268.923765:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2246 00000100:00080000:2.0:1789762268.923771:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762268.923776:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762268.923782:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762268.923797:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507619456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762268.923807:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762268.923811:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762268.923813:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762268.923817:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2246 00000100:00080000:2.0:1789762268.923822:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762268.923825:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762268.923830:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762268.923838:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762268.923850:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507619584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762268.923861:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762268.923864:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762268.923867:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762268.923872:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762268/real 1789762268] req@ffff8d25c8020a80 x1876699507619200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762268.923875:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2246 00000100:00080000:2.0:1789762268.923882:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762268.923891:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762268.923898:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762268.923911:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507619200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762268.923914:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507619712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762268.923941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762268.923942:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762268.923950:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762268.923954:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762268.923955:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762268.923965:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507619200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762268.923965:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2246 00000100:00080000:2.0:1789762268.923973:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762268.923979:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762268.923985:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762268.923998:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507619840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762268.923999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507619328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762268.924040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762268.924057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762268/real 1789762268] req@ffff8d25c8021180 x1876699507619328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762268.924071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507619328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762268.924080:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762268.924084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762268.924088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507619328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762268.924094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507619456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762268.924114:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762268.924125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762268/real 1789762268] req@ffff8d25c8022680 x1876699507619456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762268.924143:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507619456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762268.924151:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762268.924155:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762268.924158:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507619456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762268.924164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507619584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762268.924184:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762268.924195:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762268/real 1789762268] req@ffff8d25c8022a00 x1876699507619584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762268.924207:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507619584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762268.924223:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762268.924226:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762268.924229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507619584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762268.924235:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507619712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762268.924255:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762268.924266:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762268/real 1789762268] req@ffff8d25c8023800 x1876699507619712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762268.924277:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507619712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762268.924292:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762268.924296:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762268.924299:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507619712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762268.924315:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507619840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762268.924335:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762268.924346:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762268/real 1789762268] req@ffff8d25c8020000 x1876699507619840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762268.924357:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507619840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762293 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762268.924366:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762268.924369:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762268.924373:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507619840:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762268.987492:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762270.011566:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762271.035520:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762272.059614:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762273.083563:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762274.043615:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762274.043630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762274.043635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762274.043648:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2251 00000100:00080000:2.0:1789762274.043663:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762274.043671:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762274.043713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507619968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762274.043745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762274.043749:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762274.043751:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762274.043755:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2251 00000100:00080000:2.0:1789762274.043759:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762274.043762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762274.043771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762274.043784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507620096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762274.043796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762274.043799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762274.043802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762274.043808:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507619968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762274.043810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2251 00000100:00080000:2.0:1789762274.043817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762274.043839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762274.043844:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762274.043855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507620224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762274.043864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762274.043868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762274.043870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762274.043874:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2251 00000100:00080000:2.0:1789762274.043878:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762274.043880:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762274.043884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762274.043917:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762274.043926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507620352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762274.043938:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762274.043942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762274.043944:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762274.043948:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2251 00000100:00080000:2.0:1789762274.043953:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762274.043957:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762274.043962:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762274.043975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffff8d25c8022300 x1876699507619968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762274.043977:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507620480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762274.043999:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762274.044005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507619968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762274.044006:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762274.044011:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762274.044021:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2251 00000100:00080000:2.0:1789762274.044029:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762274.044034:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762274.044035:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762274.044046:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762274.044047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762274.044073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507619968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762274.044079:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507620608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762274.044107:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762274.044174:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762274.044194:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffff8d25c8020380 x1876699507620096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762274.044233:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507620096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762274.044245:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762274.044249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762274.044253:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762274.044261:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762274.044308:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762274.044320:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffff8d25c8022d80 x1876699507620224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762274.044331:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507620224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762274.044339:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762274.044342:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762274.044345:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762274.044350:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762274.044365:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762274.044374:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffff8d25c8023100 x1876699507620352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762274.044385:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507620352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762274.044392:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762274.044395:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762274.044398:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762274.044402:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507620480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762274.044416:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762274.044425:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffff8d25c8023b80 x1876699507620480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762274.044436:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507620480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762274.044443:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762274.044445:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762274.044448:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507620480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762274.044453:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507620608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762274.044466:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762274.044474:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762274/real 1789762274] req@ffff8d25c8020700 x1876699507620608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762274.044584:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507620608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762299 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762274.044594:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762274.044597:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762274.044601:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507620608:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762274.107581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762275.131592:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762276.155643:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762277.179567:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762278.203512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762279.163497:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762279.163510:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762279.163516:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762279.163527:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2256 00000100:00080000:2.0:1789762279.163538:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762279.163545:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762279.163590:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507620736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762279.163616:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762279.163619:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762279.163621:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762279.163624:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2256 00000100:00080000:2.0:1789762279.163627:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762279.163630:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762279.163641:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762279.163650:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507620864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762279.163658:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762279.163661:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762279.163663:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762279.163666:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2256 00000100:00080000:2.0:1789762279.163668:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762279.163671:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762279.163674:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762279.163683:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507620736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762279.163684:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507620992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762279.163700:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762279.163702:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762279.163703:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762279.163705:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2256 00000100:00080000:2.0:1789762279.163707:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762279.163709:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762279.163712:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762279.163722:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507621120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762279.163730:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762279.163732:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762279.163734:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762279.163736:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2256 00000100:00080000:2.0:1789762279.163738:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762279.163740:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762279.163743:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762279.163750:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507621248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762279.163756:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762279.163759:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762279.163761:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762279.163764:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2256 00000100:00080000:2.0:1789762279.163766:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762279.163768:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762279.163770:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762279.163778:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762279.163780:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507621376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762279.163840:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffff8d25c8021880 x1876699507620736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762279.163861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507620736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762279.163873:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762279.163877:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762279.163880:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507620736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762279.163888:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762279.163912:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762279.163926:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffff8d25c8021c00 x1876699507620864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762279.163937:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507620864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762279.163946:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762279.163949:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762279.163953:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762279.163984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762279.164005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762279.164015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffff8d25c8021500 x1876699507620992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762279.164023:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507620992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762279.164029:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762279.164032:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762279.164034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507620992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762279.164038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762279.164050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762279.164056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffff8d25c8020e00 x1876699507621120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762279.164064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507621120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762279.164069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762279.164071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762279.164073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762279.164076:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507621248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762279.164087:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762279.164093:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffff8d25c8023480 x1876699507621248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762279.164101:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507621248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762279.164106:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762279.164108:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762279.164110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507621248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762279.164113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507621376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762279.164124:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762279.164130:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762279/real 1789762279] req@ffff8d25c8020700 x1876699507621376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762279.164138:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507621376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762304 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762279.164143:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762279.164144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762279.164146:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507621376:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762279.227571:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762280.251540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762281.275590:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762282.299575:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762283.323537:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762284.283492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.283500:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762284.283504:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.283514:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2261 00000100:00080000:2.0:1789762284.283524:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.283532:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762284.283580:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507621504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762284.283610:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.283612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762284.283614:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.283617:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2261 00000100:00080000:2.0:1789762284.283620:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.283623:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762284.283632:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762284.283642:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507621632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762284.283652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507621504:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762284.283652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.283660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762284.283663:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.283666:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2261 00000100:00080000:2.0:1789762284.283669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.283673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762284.283678:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762284.283691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507621760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762284.283703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.283707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762284.283709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762284.283715:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762284.283719:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2261 00000100:00080000:2.0:1789762284.283725:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.283729:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762284.283734:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762284.283744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507621888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762284.283752:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762284.283755:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762284] req@ffff8d25c8023b80 x1876699507621504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762284.283756:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762284.283760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.283767:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2261 00000100:00080000:2.0:1789762284.283773:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.283777:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762284.283781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762284.283785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507621504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762284.283792:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507622016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762284.283812:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762284.283820:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762284.283822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762284.283826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762284.283828:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507621504:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762284.283829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762284.283836:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2261 00000100:00080000:2.0:1789762284.283839:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762284.283840:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762284.283843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762284.283853:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507622144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762284.283874:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762284.283899:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762284.283909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762284] req@ffff8d25c8023100 x1876699507621632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762284.283918:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507621632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762284.283923:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762284.283926:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762284.283928:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762284.283932:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762284.283942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762284.283949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762284] req@ffff8d25c8022d80 x1876699507621760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762284.283956:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507621760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762284.283961:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762284.283963:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762284.283964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762284.283968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762284.283979:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762284.283984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762284] req@ffff8d25c8020380 x1876699507621888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762284.283992:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507621888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762284.284006:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762284.284010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762284.284013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507621888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762284.284019:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762284.284039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762284.284051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762284] req@ffff8d25c8022300 x1876699507622016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762284.284061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507622016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762284.284067:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762284.284069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762284.284071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762284.284074:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762284.284088:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762284.284094:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762284/real 1789762284] req@ffff8d25c8020000 x1876699507622144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762284.284102:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507622144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762309 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762284.284107:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762284.284108:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762284.284110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622144:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762284.347533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762285.371635:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762286.395563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762287.419565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762288.443549:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762289.403562:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762289.403574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762289.403579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762289.403590:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2267 00000100:00080000:2.0:1789762289.403603:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762289.403609:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762289.403645:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507622272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762289.403673:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762289.403676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762289.403678:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762289.403681:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2267 00000100:00080000:2.0:1789762289.403684:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762289.403687:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762289.403695:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762289.403705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507622400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762289.403714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762289.403717:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762289.403718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762289.403722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2267 00000100:00080000:2.0:1789762289.403724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762289.403727:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762289.403730:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762289.403739:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507622528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762289.403748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762289.403753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:0.0:1789762289.403755:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507622272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762289.403755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762289.403759:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2267 00000100:00080000:2.0:1789762289.403764:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762289.403771:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762289.403775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762289.403785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507622656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762289.403793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762289.403796:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762289.403798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762289.403800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2267 00000100:00080000:2.0:1789762289.403804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762289.403808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762289.403812:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762289.403819:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507622784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762289.403826:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762289.403829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762289.403830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762289.403837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2267 00000800:00000100:0.0:1789762289.403838:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762289.403843:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762289.403848:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762289.403852:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762289.403863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507622912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762289.403889:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762289/real 1789762289] req@ffff8d25c8023800 x1876699507622272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762289.403907:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507622272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762289.403916:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762289.403920:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762289.403923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507622272:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762289.403929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507622400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762289.403948:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762289.403957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762289/real 1789762289] req@ffff8d25c8022a00 x1876699507622400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762289.403967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507622400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762289.403979:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762289.403981:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762289.403984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507622400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762289.404003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507622528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762289.404016:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762289.404024:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762289/real 1789762289] req@ffff8d25c8022680 x1876699507622528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762289.404033:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507622528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762289.404039:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762289.404041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762289.404044:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507622528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762289.404048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507622656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762289.404059:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762289.404066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762289/real 1789762289] req@ffff8d25c8021180 x1876699507622656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762289.404075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507622656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762289.404081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762289.404083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762289.404085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507622656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762289.404090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762289.404105:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762289.404118:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762289/real 1789762289] req@ffff8d25c8020a80 x1876699507622784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762289.404134:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507622784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762289.404142:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762289.404146:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762289.404150:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762289.404156:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762289.404177:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762289.404190:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762289/real 1789762289] req@ffff8d25c8020000 x1876699507622912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762289.404203:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507622912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762314 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762289.404223:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762289.404226:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762289.404229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507622912:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762289.467625:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762290.491595:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762291.515625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762292.539509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762293.563583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762294.523500:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762294.523512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762294.523517:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762294.523529:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2272 00000100:00080000:2.0:1789762294.523540:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762294.523548:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762294.523593:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507623040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762294.523624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762294.523628:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762294.523631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762294.523636:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2272 00000100:00080000:2.0:1789762294.523641:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762294.523647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762294.523668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762294.523676:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507623040:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762294.523690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507623168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762294.523705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762294.523708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762294.523710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762294.523725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2272 00000100:00080000:2.0:1789762294.523728:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762294.523732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762294.523737:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762294.523748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507623296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762294.523755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762294.523759:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762294.523760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762294.523763:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2272 00000100:00080000:2.0:1789762294.523767:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762294.523769:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762294.523773:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762294.523781:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762294.523783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507623424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762294.523794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762294.523797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762294.523799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762294.523802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2272 00000100:00080000:2.0:1789762294.523804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762294.523807:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762294.523810:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762294.523818:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507623552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762294.523824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762294.523827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762294.523829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762294.523831:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2272 00000100:00080000:2.0:1789762294.523834:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762294.523837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762294.523852:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762294.523858:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762294/real 1789762294] req@ffff8d25c8022300 x1876699507623040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762294.523863:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507623680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762294.523890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507623040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762294.523901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762294.523906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762294.523910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507623040:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762294.523935:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762294.523954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762294.523971:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762294/real 1789762294] req@ffff8d25c8020380 x1876699507623168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762294.523981:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507623168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762294.523988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762294.523990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762294.523992:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762294.523996:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762294.524020:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762294.524028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762294/real 1789762294] req@ffff8d25c8022d80 x1876699507623296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762294.524036:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507623296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762294.524048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762294.524052:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762294.524055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762294.524061:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762294.524082:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762294.524095:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762294/real 1789762294] req@ffff8d25c8023100 x1876699507623424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762294.524108:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507623424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762294.524114:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762294.524117:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762294.524130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762294.524134:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507623552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762294.524150:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762294.524158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762294/real 1789762294] req@ffff8d25c8023b80 x1876699507623552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762294.524165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507623552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762294.524171:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762294.524173:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762294.524176:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507623552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762294.524180:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507623680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762294.524198:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762294.524204:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762294/real 1789762294] req@ffff8d25c8020700 x1876699507623680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762294.524212:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507623680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762319 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762294.524217:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762294.524219:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762294.524221:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507623680:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762294.587608:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762295.611614:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762296.635561:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762297.659563:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762298.683576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762299.643492:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762299.643503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762299.643508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762299.643521:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2277 00000100:00080000:2.0:1789762299.643534:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762299.643543:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762299.643596:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507623808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762299.643632:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762299.643636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762299.643638:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762299.643643:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2277 00000100:00080000:2.0:1789762299.643647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762299.643651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762299.643664:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762299.643683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507623936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762299.643692:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507623808:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762299.643694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762299.643700:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762299.643703:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762299.643708:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2277 00000100:00080000:2.0:1789762299.643711:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762299.643714:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762299.643718:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762299.643728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507624064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762299.643735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762299.643738:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762299.643739:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762299.643741:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2277 00000100:00080000:2.0:1789762299.643745:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762299.643747:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762299.643750:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762299.643757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507624192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762299.643764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762299.643766:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762299.643768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762299.643771:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762299.643772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2277 00000100:00080000:2.0:1789762299.643777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762299.643780:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762299.643784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762299.643798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507624320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762299.643806:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762299.643808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762299.643810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762299.643813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2277 00000100:00080000:2.0:1789762299.643815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762299.643818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762299.643821:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762299.643822:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762299/real 1789762299] req@ffff8d25c8023480 x1876699507623808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762299.643832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507624448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762299.643843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507623808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762299.643855:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762299.643858:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762299.643861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507623808:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762299.643885:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762299.643901:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762299.643910:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762299/real 1789762299] req@ffff8d25c8020e00 x1876699507623936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762299.643919:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507623936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762299.643925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762299.643927:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762299.643930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507623936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762299.643933:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762299.643943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762299.643969:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762299/real 1789762299] req@ffff8d25c8021500 x1876699507624064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762299.643983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507624064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762299.643992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762299.643996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762299.644000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762299.644007:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762299.644045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762299.644055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762299/real 1789762299] req@ffff8d25c8021c00 x1876699507624192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762299.644063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507624192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762299.644069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762299.644071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762299.644074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762299.644077:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507624320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762299.644090:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762299.644096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762299/real 1789762299] req@ffff8d25c8021880 x1876699507624320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762299.644103:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507624320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762299.644109:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762299.644111:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762299.644113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507624320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762299.644116:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507624448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762299.644126:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762299.644132:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762299/real 1789762299] req@ffff8d25c8020700 x1876699507624448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762299.644140:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507624448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762324 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762299.644145:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762299.644147:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762299.644149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507624448:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762299.707540:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762300.731560:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762301.755599:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762302.779568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762303.803544:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762304.763510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762304.763520:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762304.763535:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762304.763544:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2282 00000100:00080000:2.0:1789762304.763553:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762304.763561:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762304.763594:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507624576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762304.763624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762304.763629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762304.763633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762304.763639:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2282 00000100:00080000:2.0:1789762304.763645:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762304.763651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762304.763661:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762304.763667:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507624576:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762304.763679:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507624704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762304.763711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762304.763714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762304.763716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762304.763720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2282 00000100:00080000:2.0:1789762304.763724:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762304.763727:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762304.763731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762304.763742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507624832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762304.763750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762304.763753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762304.763757:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762304.763758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762304.763763:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2282 00000100:00080000:2.0:1789762304.763767:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762304.763775:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762304.763780:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762304.763791:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507624960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762304.763799:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762304.763802:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762304.763804:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762304.763810:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762304/real 1789762304] req@ffff8d25c8023b80 x1876699507624576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762304.763811:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2282 00000100:00080000:2.0:1789762304.763816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762304.763822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762304.763830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762304.763844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507625088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762304.763851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507624576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762304.763855:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762304.763861:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762304.763864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762304.763873:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2282 00000100:00080000:2.0:1789762304.763879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762304.763884:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762304.763884:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762304.763889:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762304.763895:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762304.763901:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507625216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762304.763906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507624576:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762304.763942:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762304.763970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762304.763986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762304/real 1789762304] req@ffff8d25c8023100 x1876699507624704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762304.764002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507624704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762304.764014:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762304.764018:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762304.764022:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762304.764030:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762304.764051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762304.764067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762304/real 1789762304] req@ffff8d25c8022d80 x1876699507624832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762304.764083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507624832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762304.764094:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762304.764098:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762304.764103:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762304.764111:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762304.764137:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762304.764154:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762304/real 1789762304] req@ffff8d25c8020380 x1876699507624960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762304.764174:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507624960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762304.764187:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762304.764192:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762304.764197:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507624960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762304.764206:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762304.764233:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762304.764248:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762304/real 1789762304] req@ffff8d25c8022300 x1876699507625088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762304.764263:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507625088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762304.764273:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762304.764277:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762304.764281:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762304.764305:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762304.764332:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762304.764346:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762304/real 1789762304] req@ffff8d25c8020000 x1876699507625216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762304.764362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507625216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762329 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762304.764372:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762304.764420:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762304.764425:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625216:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762304.827578:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762305.851603:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762306.875542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762307.899544:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762308.923654:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762309.883543:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762309.883555:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762309.883562:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762309.883595:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2287 00000100:00080000:3.0:1789762309.883606:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762309.883625:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762309.883689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507625344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762309.883716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762309.883720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762309.883722:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762309.883727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2287 00000100:00080000:3.0:1789762309.883731:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762309.883735:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762309.883761:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762309.883781:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507625472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762309.883793:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507625344:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762309.883802:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762309.883809:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762309.883812:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762309.883820:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2287 00000100:00080000:3.0:1789762309.883826:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762309.883831:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762309.883837:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762309.883850:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507625600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762309.883859:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762309.883863:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762309.883865:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762309.883873:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2287 00000800:00000100:0.0:1789762309.883876:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762309.883878:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762309.883885:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762309.883890:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762309.883901:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507625728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762309.883921:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762309.883923:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762309.883925:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762309.883927:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2287 00000100:00080000:3.0:1789762309.883930:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762309.883933:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762309.883937:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762309.883947:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507625856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762309.883956:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762309.883962:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762309.883963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762309/real 1789762309] req@ffff8d25d1b0a680 x1876699507625344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762309.883964:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762309.883972:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2287 00000100:00080000:3.0:1789762309.883977:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762309.883984:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762309.884004:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762309.884014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507625344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762309.884025:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507625984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762309.884039:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762309.884051:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762309.884071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507625344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762309.884105:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507625472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762309.884163:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762309.884181:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762309/real 1789762309] req@ffff8d25d1b08700 x1876699507625472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762309.884196:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507625472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762309.884206:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762309.884210:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762309.884214:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507625472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762309.884221:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507625600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762309.884284:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762309.884330:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762309/real 1789762309] req@ffff8d25d1b0b800 x1876699507625600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762309.884347:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507625600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762309.884358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762309.884363:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762309.884368:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507625600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762309.884376:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507625728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762309.884458:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762309.884475:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762309/real 1789762309] req@ffff8d25d1b08e00 x1876699507625728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762309.884489:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507625728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762309.884499:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762309.884502:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762309.884505:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507625728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762309.884512:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762309.884532:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762309.884542:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762309/real 1789762309] req@ffff8d25d1b09880 x1876699507625856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762309.884555:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507625856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762309.884563:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762309.884566:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762309.884569:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762309.884593:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762309.884625:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762309.884636:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762309/real 1789762309] req@ffff8d25d1b0bb80 x1876699507625984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762309.884665:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507625984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762334 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762309.884674:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762309.884677:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762309.884681:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507625984:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762309.947582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762310.971590:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762311.995706:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762313.019605:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762314.043549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762315.003529:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762315.003540:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762315.003545:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762315.003557:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2292 00000100:00080000:0.0:1789762315.003568:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762315.003575:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762315.003621:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507626112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762315.003653:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762315.003657:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762315.003658:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762315.003662:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2292 00000100:00080000:0.0:1789762315.003665:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762315.003668:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762315.003677:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762315.003688:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507626240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762315.003698:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762315.003701:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762315.003703:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762315.003710:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2292 00000100:00100000:2.0:1789762315.003711:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507626112:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762315.003713:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762315.003720:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762315.003727:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762315.003734:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507626368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762315.003740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762315.003743:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762315.003745:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762315.003748:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2292 00000100:00080000:0.0:1789762315.003751:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762315.003753:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762315.003757:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762315.003768:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507626496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762315.003776:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762315.003779:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762315.003781:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762315.003783:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2292 00000100:00080000:0.0:1789762315.003786:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762315.003789:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762315.003792:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762315.003801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507626624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762315.003810:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762315.003817:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762315.003823:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762315.003826:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762315.003833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2292 00000100:00080000:0.0:1789762315.003841:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762315.003847:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762315.003853:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762315.003871:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffff8d25d25b8a80 x1876699507626112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762315.003872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507626752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762315.003920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507626112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762315.003931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762315.003935:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762315.003939:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507626112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762315.003965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507626240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762315.003987:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762315.003998:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffff8d25d25ba300 x1876699507626240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762315.004013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507626240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762315.004023:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762315.004027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762315.004031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507626240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762315.004038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507626368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762315.004063:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762315.004075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffff8d25d25b8000 x1876699507626368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762315.004085:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507626368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762315.004092:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762315.004094:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762315.004097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507626368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762315.004101:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507626496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762315.004116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762315.004123:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffff8d25d25ba680 x1876699507626496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762315.004131:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507626496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762315.004137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762315.004140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762315.004142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507626496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762315.004145:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507626624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762315.004158:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762315.004165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffff8d25d25bb800 x1876699507626624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762315.004173:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507626624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762315.004180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762315.004182:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762315.004184:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507626624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762315.004188:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507626752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762315.004199:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762315.004206:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762315/real 1789762315] req@ffff8d25d25b9f80 x1876699507626752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762315.004228:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507626752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762340 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762315.004234:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762315.004237:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762315.004240:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507626752:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762315.067617:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762316.091615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762317.115555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762318.139508:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762319.163581:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762320.123491:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762320.123501:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762320.123505:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762320.123515:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2297 00000100:00080000:1.0:1789762320.123525:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762320.123533:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762320.123603:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507626880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762320.123626:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762320.123629:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762320.123631:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762320.123651:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2297 00000100:00080000:1.0:1789762320.123654:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762320.123658:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762320.123667:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762320.123679:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507627008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762320.123695:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762320.123701:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507626880:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762320.123702:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762320.123706:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762320.123716:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2297 00000100:00080000:1.0:1789762320.123718:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762320.123721:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762320.123725:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762320.123737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507627136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762320.123746:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762320.123749:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762320.123751:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762320.123753:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2297 00000100:00080000:1.0:1789762320.123757:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762320.123760:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762320.123763:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762320.123774:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507627264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762320.123783:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762320.123792:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762320.123796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762320.123798:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762320.123802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2297 00000100:00080000:1.0:1789762320.123808:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762320.123821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762320.123825:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762320.123837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507627392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762320.123843:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffff8d25c816aa00 x1876699507626880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762320.123848:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762320.123859:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762320.123863:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762320.123872:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2297 00000100:00080000:1.0:1789762320.123885:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762320.123909:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762320.123911:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507626880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762320.123915:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762320.123933:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507627520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762320.123946:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762320.123953:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762320.123976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507626880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762320.124013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762320.124053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762320.124073:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffff8d25c816a680 x1876699507627008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762320.124084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507627008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762320.124093:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762320.124096:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762320.124099:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762320.124104:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762320.124119:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762320.124147:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffff8d25c8169f80 x1876699507627136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762320.124157:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507627136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762320.124163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762320.124165:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762320.124168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762320.124173:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762320.124209:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762320.124219:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffff8d25c8168e00 x1876699507627264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762320.124228:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507627264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762320.124246:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762320.124248:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762320.124251:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762320.124255:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507627392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762320.124268:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762320.124280:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffff8d25c8168380 x1876699507627392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762320.124295:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507627392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762320.124316:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762320.124320:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762320.124324:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507627392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762320.124332:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507627520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762320.124357:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762320.124368:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762320/real 1789762320] req@ffff8d25c8168700 x1876699507627520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762320.124409:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507627520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762345 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762320.124417:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762320.124419:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762320.124422:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507627520:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762320.187553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762321.211524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762322.235536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762323.259490:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762324.283604:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762325.243473:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762325.243483:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762325.243488:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762325.243501:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2302 00000100:00080000:1.0:1789762325.243513:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762325.243523:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762325.243559:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507627648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762325.243588:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762325.243594:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762325.243596:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762325.243601:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2302 00000100:00080000:1.0:1789762325.243610:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762325.243616:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762325.243625:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762325.243641:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507627648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762325.243643:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507627776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762325.243666:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762325.243671:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762325.243673:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762325.243679:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2302 00000100:00080000:1.0:1789762325.243683:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762325.243687:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762325.243692:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762325.243708:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507627904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762325.243720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762325.243724:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762325.243726:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762325.243730:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2302 00000800:00000100:2.0:1789762325.243736:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762325.243746:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762325.243754:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762325.243763:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762325.243781:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507628032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762325.243793:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffff8d25c8169500 x1876699507627648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762325.243794:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762325.243808:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762325.243811:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762325.243827:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2302 00000100:00080000:1.0:1789762325.243833:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762325.243840:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507627648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762325.243841:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762325.243861:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789762325.243865:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762325.243870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762325.243874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507627648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762325.243883:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507628160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762325.243897:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762325.243902:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762325.243906:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762325.243908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627776:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762325.243922:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2302 00000100:00080000:1.0:1789762325.243927:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762325.243932:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762325.243939:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762325.243950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762325.243956:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507628288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762325.243966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffff8d25c8168000 x1876699507627776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762325.244002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507627776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762325.244008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762325.244012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762325.244015:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762325.244021:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762325.244034:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762325.244042:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffff8d25c816b480 x1876699507627904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762325.244050:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507627904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762325.244055:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762325.244057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762325.244059:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507627904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762325.244063:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762325.244077:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762325.244086:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffff8d25c8169880 x1876699507628032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762325.244098:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507628032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762325.244115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762325.244119:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762325.244122:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762325.244130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507628160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762325.244161:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762325.244170:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffff8d25c816a300 x1876699507628160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762325.244178:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507628160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762325.244184:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762325.244186:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762325.244188:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507628160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762325.244203:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507628288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762325.244216:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762325.244223:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762325/real 1789762325] req@ffff8d25c816ad80 x1876699507628288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762325.244230:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507628288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762350 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762325.244235:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762325.244237:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762325.244240:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507628288:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762325.307545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762326.331641:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762327.355527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762328.379535:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762329.403507:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762330.363438:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762330.363447:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762330.363451:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762330.363460:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2308 00000100:00080000:1.0:1789762330.363469:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762330.363477:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762330.363515:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507628416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762330.363543:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762330.363546:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762330.363547:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762330.363551:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2308 00000100:00080000:1.0:1789762330.363554:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762330.363557:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762330.363566:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762330.363579:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507628544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762330.363588:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762330.363590:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762330.363592:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762330.363602:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2308 00000100:00100000:2.0:1789762330.363606:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507628416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762330.363607:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762330.363622:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762330.363626:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762330.363636:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507628672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762330.363648:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762330.363651:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762330.363653:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762330.363657:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2308 00000100:00080000:1.0:1789762330.363662:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762330.363668:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762330.363673:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762330.363689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507628800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762330.363701:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762330.363704:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762330.363705:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762330.363709:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2308 00000100:00080000:1.0:1789762330.363712:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762330.363716:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762330.363720:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762330.363732:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507628928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762330.363746:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762330.363748:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628544:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762330.363751:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762330.363757:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762330.363762:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2308 00000100:00080000:1.0:1789762330.363766:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762330.363770:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762330.363775:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762330.363789:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762330.363797:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507629056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762330.363813:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762330.363849:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507628928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762330.363868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507629056:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762330.427557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762331.451516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762332.475506:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789762333.435823:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789762333.435899:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762333.435917:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762333.435971:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762333.435982:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762333.435990:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762333.436000:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762333.436008:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789762333.436109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762333] req@ffff8d25c8169880 x1876699507628928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762333.436137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507628928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762333.436150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762333.436154:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762333.436159:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507628928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762333.436167:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762333] req@ffff8d25c816b480 x1876699507629056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762333.436185:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507629056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762333.436192:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762333.436195:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762333.436198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507629056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762333.436204:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762333] req@ffff8d25c8169c00 x1876699507628544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762333.436220:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507628544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762333.436227:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762333.436230:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762333.436234:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762333.436240:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762333] req@ffff8d25c816ad80 x1876699507628672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762333.436252:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507628672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762333.436260:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762333.436264:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762333.436267:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762333.436274:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762333] req@ffff8d25c816a300 x1876699507628800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762333.436286:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507628800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762333.436295:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762333.436303:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762333.436307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507628800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762333.436314:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762330/real 1789762333] req@ffff8d25c8168a80 x1876699507628416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762333.436327:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507628416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762355 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762333.436335:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762333.436338:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762333.436341:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507628416:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789762333.499486:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762334.523520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762335.483589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762335.483602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762335.483606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762335.483618:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2313 00000100:00080000:3.0:1789762335.483630:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762335.483655:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762335.483694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507629184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762335.483742:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762335.483746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762335.483747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762335.483751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2313 00000100:00080000:3.0:1789762335.483755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762335.483763:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762335.483772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762335.483786:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507629312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762335.483801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762335.483803:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507629184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762335.483810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762335.483816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762335.483822:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2313 00000100:00080000:3.0:1789762335.483827:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762335.483832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762335.483840:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762335.483858:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507629440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762335.483872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762335.483877:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762335.483879:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762335.483884:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2313 00000100:00080000:3.0:1789762335.483889:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789762335.483893:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762335.483899:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762335.483909:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762335.483926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507629568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762335.483939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762335.483943:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789762335.483946:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762335/real 1789762335] req@ffff8d25d1b08a80 x1876699507629184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762335.483948:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762335.483960:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2313 00000100:00080000:3.0:1789762335.483969:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762335.483977:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762335.483984:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762335.483987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507629184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762335.484003:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507629696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762335.484004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762335.484026:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762335.484034:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762335.484038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507629184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762335.484040:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762335.484046:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762335.484055:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2313 00000100:00080000:3.0:1789762335.484078:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762335.484085:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762335.484092:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762335.484094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507629312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762335.484110:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507629824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762335.484145:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762335.484169:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762335/real 1789762335] req@ffff8d25d1b0aa00 x1876699507629312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762335.484186:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507629312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762335.484197:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762335.484201:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762335.484205:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507629312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762335.484212:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507629440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762335.484240:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762335.484254:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762335/real 1789762335] req@ffff8d25d1b09180 x1876699507629440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762335.484268:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507629440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762335.484278:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762335.484282:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762335.484286:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507629440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762335.484293:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507629568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762335.484319:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762335.484335:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762335/real 1789762335] req@ffff8d25d1b0ad80 x1876699507629568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762335.484349:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507629568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762335.484359:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762335.484362:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762335.484366:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507629568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762335.484373:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507629696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762335.484441:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762335.484457:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762335/real 1789762335] req@ffff8d25d1b09c00 x1876699507629696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762335.484473:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507629696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762335.484484:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762335.484488:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762335.484492:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507629696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762335.484521:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507629824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762335.484548:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762335.484562:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762335/real 1789762335] req@ffff8d25d1b0b100 x1876699507629824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762335.484577:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507629824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762360 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762335.484587:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762335.484591:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762335.484595:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507629824:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762335.547543:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762336.571488:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762337.595587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762338.619518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762339.643524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762340.603583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762340.603594:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762340.603599:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762340.603623:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2318 00000100:00080000:3.0:1789762340.603636:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762340.603646:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762340.603702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507629952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762340.603761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762340.603765:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762340.603767:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762340.603772:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2318 00000100:00080000:3.0:1789762340.603776:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762340.603782:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762340.603801:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762340.603815:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507629952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762340.603823:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507630080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762340.603835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762340.603838:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762340.603840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762340.603844:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2318 00000100:00080000:3.0:1789762340.603846:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762340.603850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762340.603854:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762340.603864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507630208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762340.603872:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762340.603875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762340.603876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789762340.603895:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762340.603895:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2318 00000100:00080000:3.0:1789762340.603903:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762340.603955:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762340/real 1789762340] req@ffff8d25d1b0b480 x1876699507629952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762340.603985:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507629952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762340.603996:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762340.604005:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762340.604009:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762340.604015:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507629952:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789762340.604038:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762340.604077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507630336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762340.604078:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762340.604101:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762340.604113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762340/real 1789762340] req@ffff8d25d1b09500 x1876699507630080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762340.604124:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507630080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762340.604131:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762340.604134:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762340.604138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762340.604143:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762340.604159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762340.604168:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762340/real 1789762340] req@ffff8d25d1b08000 x1876699507630208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762340.604179:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507630208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762340.604198:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762340.604200:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762340.604203:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762340.604230:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762340.604235:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762340.604237:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762340.604242:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2318 00000100:00080000:3.0:1789762340.604249:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762340.604254:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762340.604261:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762340.604301:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762340.604360:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507630464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762340.604373:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762340.604406:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762340.604413:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762340.604415:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762340.604421:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2318 00000100:00080000:3.0:1789762340.604428:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762340.604432:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762340.604449:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762340.604465:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762340/real 1789762340] req@ffff8d25d1b08380 x1876699507630336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762340.604470:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507630592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762340.604528:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507630336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762340.604540:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762340.604547:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762340.604551:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762340.604594:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507630464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762340.604614:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762340.604623:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762340/real 1789762340] req@ffff8d25d1b0a300 x1876699507630464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762340.604648:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507630464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762340.604655:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762340.604658:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762340.604661:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507630464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762340.604665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507630592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762340.604679:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762340.604687:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762340/real 1789762340] req@ffff8d25d1b08000 x1876699507630592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762340.604695:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507630592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762365 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762340.604721:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762340.604723:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762340.604726:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507630592:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762340.667590:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762341.691566:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762342.715520:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762343.739620:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762344.763478:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762345.723517:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762345.723531:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762345.723536:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762345.723550:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2323 00000100:00080000:3.0:1789762345.723562:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762345.723571:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762345.723619:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507630720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762345.723672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762345.723679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762345.723683:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762345.723691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2323 00000100:00080000:3.0:1789762345.723698:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762345.723706:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762345.723720:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762345.723743:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507630720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762345.723748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507630848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762345.723773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762345.723780:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762345.723783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762345.723790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2323 00000100:00080000:3.0:1789762345.723799:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762345.723806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762345.723815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762345.723834:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762345.723842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507630976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762345.723861:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762345.723868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762345.723872:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762345.723880:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762345/real 1789762345] req@ffff8d25d1b09500 x1876699507630720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762345.723882:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2323 00000100:00080000:3.0:1789762345.723895:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762345.723905:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762345.723911:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507630720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762345.723919:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1789762345.723927:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762345.723933:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762345.723938:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507630720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762345.723948:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507631104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762345.723969:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762345.723976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762345.723979:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762345.724006:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762345.724013:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2323 00000100:00080000:3.0:1789762345.724025:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762345.724033:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762345.724042:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762345.724050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762345.724076:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762345/real 1789762345] req@ffff8d25d1b0b480 x1876699507630848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762345.724077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507631232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762345.724124:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507630848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762345.724132:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762345.724139:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762345.724141:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762345.724147:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762345.724150:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762345.724156:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762345.724164:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2323 00000100:00100000:0.0:1789762345.724170:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630976:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762345.724176:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762345.724187:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762345.724197:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762345.724202:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762345.724220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762345/real 1789762345] req@ffff8d25d1b0b100 x1876699507630976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762345.724228:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507631360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762345.724242:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507630976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762345.724274:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762345.724278:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762345.724282:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507630976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762345.724291:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762345.724311:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762345.724322:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762345/real 1789762345] req@ffff8d25d1b09c00 x1876699507631104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762345.724334:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507631104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762345.724342:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762345.724345:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762345.724348:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762345.724366:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507631232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762345.724411:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762345.724424:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762345/real 1789762345] req@ffff8d25d1b0ad80 x1876699507631232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762345.724436:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507631232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762345.724445:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762345.724448:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762345.724452:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507631232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762345.724457:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507631360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762345.724475:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762345.724484:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762345/real 1789762345] req@ffff8d25d1b09180 x1876699507631360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762345.724496:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507631360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762370 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762345.724508:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762345.724513:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762345.724518:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507631360:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762345.787637:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762346.811592:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762347.835631:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762348.859620:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762349.883522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762350.843487:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762350.843496:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762350.843500:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762350.843508:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2328 00000100:00080000:2.0:1789762350.843516:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762350.843522:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762350.843555:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507631488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762350.843589:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762350.843592:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762350.843594:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762350.843597:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2328 00000100:00080000:2.0:1789762350.843600:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762350.843604:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762350.843613:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762350.843625:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507631616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762350.843629:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507631488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762350.843643:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762350.843647:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762350.843648:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762350.843652:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2328 00000100:00080000:2.0:1789762350.843654:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762350.843657:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762350.843661:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762350.843669:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507631744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762350.843676:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762350.843679:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762350.843681:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762350.843683:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2328 00000100:00080000:2.0:1789762350.843686:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762350.843688:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762350.843692:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762350.843700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507631872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762350.843705:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762350.843715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762350.843718:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762350.843720:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762350.843725:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2328 00000100:00080000:2.0:1789762350.843740:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762350.843743:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762350.843746:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762350.843758:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507632000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762350.843764:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762350/real 1789762350] req@ffff8d25c8020a80 x1876699507631488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762350.843770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762350.843779:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762350.843782:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762350.843793:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2328 00000100:00080000:2.0:1789762350.843795:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762350.843809:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762350.843816:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762350.843818:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507631488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762350.843830:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762350.843837:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762350.843839:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507632128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762350.843842:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507631488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762350.843869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762350.843890:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762350.843899:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762350/real 1789762350] req@ffff8d25c8021180 x1876699507631616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762350.843920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507631616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762350.843927:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762350.843929:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762350.843932:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762350.843935:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762350.843964:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762350.843973:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762350/real 1789762350] req@ffff8d25c8022680 x1876699507631744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762350.843982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507631744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762350.843993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762350.843995:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762350.843998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762350.844002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762350.844016:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762350.844022:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762350/real 1789762350] req@ffff8d25c8022a00 x1876699507631872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762350.844031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507631872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762350.844037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762350.844039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762350.844041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507631872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762350.844045:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762350.844055:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762350.844062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762350/real 1789762350] req@ffff8d25c8023800 x1876699507632000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762350.844070:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507632000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762350.844075:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762350.844078:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762350.844080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762350.844084:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762350.844095:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762350.844102:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762350/real 1789762350] req@ffff8d25c8020000 x1876699507632128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762350.844110:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507632128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762375 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762350.844115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762350.844117:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762350.844119:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632128:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762350.907503:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762351.931562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762352.955487:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762353.979557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762355.003493:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762355.963482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762355.963492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762355.963496:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762355.963505:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2333 00000100:00080000:3.0:1789762355.963513:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762355.963519:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762355.963552:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507632256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762355.963576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762355.963579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762355.963581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762355.963585:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2333 00000100:00080000:3.0:1789762355.963588:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762355.963602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762355.963610:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762355.963620:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507632384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762355.963630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762355.963632:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762355.963635:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762355.963638:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507632256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762355.963641:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2333 00000100:00080000:3.0:1789762355.963647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762355.963651:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762355.963654:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762355.963664:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507632512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762355.963671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762355.963674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762355.963676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762355.963679:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2333 00000100:00080000:3.0:1789762355.963683:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762355.963685:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762355.963688:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762355.963696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507632640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762355.963704:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762355.963710:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000800:00000100:0.0:1789762355.963711:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762355.963712:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762355.963720:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2333 00000100:00080000:3.0:1789762355.963722:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762355.963725:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762355.963729:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762355.963737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507632768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762355.963743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762355.963746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762355.963748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762355.963752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2333 00000100:00080000:3.0:1789762355.963755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762355.963758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762355.963762:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762355.963764:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762355/real 1789762355] req@ffff8d25d1b0aa00 x1876699507632256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762355.963772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507632896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762355.963792:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507632256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762355.963802:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762355.963806:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762355.963809:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507632256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762355.963833:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507632384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762355.963854:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762355.963864:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762355/real 1789762355] req@ffff8d25d1b08a80 x1876699507632384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762355.963874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507632384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762355.963880:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762355.963883:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762355.963885:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507632384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762355.963889:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507632512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762355.963901:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762355.963928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762355/real 1789762355] req@ffff8d25d1b0bb80 x1876699507632512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762355.963938:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507632512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762355.963962:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762355.963964:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762355.963967:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507632512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762355.963971:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507632640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762355.963988:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762355.963996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762355/real 1789762355] req@ffff8d25d1b09880 x1876699507632640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762355.964013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507632640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762355.964023:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762355.964027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762355.964030:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507632640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762355.964037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762355.964060:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762355.964073:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762355/real 1789762355] req@ffff8d25d1b08e00 x1876699507632768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762355.964083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507632768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762355.964088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762355.964091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762355.964093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762355.964097:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762355.964112:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762355.964120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762355/real 1789762355] req@ffff8d25d1b0b800 x1876699507632896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762355.964128:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507632896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762380 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762355.964150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762355.964152:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762355.964154:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507632896:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762356.027534:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762357.051544:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762358.075504:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762359.099516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762360.123487:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762361.083496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762361.083504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762361.083508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762361.083516:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2338 00000100:00080000:0.0:1789762361.083524:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762361.083529:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762361.083560:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507633024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762361.083581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762361.083584:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762361.083585:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762361.083588:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2338 00000100:00080000:0.0:1789762361.083591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762361.083593:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762361.083601:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762361.083613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507633152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762361.083620:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762361.083624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762361.083625:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762361.083628:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2338 00000100:00080000:0.0:1789762361.083630:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762361.083635:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507633024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762361.083636:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762361.083643:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762361.083650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507633280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762361.083657:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762361.083660:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762361.083661:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762361.083663:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2338 00000100:00080000:0.0:1789762361.083666:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762361.083668:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762361.083681:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762361.083689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507633408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762361.083695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762361.083698:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762361.083699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762361.083701:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2338 00000100:00080000:0.0:1789762361.083704:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762361.083708:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762361.083711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762361.083716:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762361.083742:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507633536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762361.083750:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762361.083753:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762361.083755:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762361.083759:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2338 00000100:00080000:0.0:1789762361.083762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762361.083766:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762361.083770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762361.083772:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffff8d25d25b9180 x1876699507633024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762361.083783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507633664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762361.083795:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507633024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.083806:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.083820:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762361.083825:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507633024:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762361.083831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762361.083851:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762361.083862:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffff8d25d25bbb80 x1876699507633152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.083912:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507633152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.083922:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.083926:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762361.083930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762361.083957:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762361.083975:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762361.083983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffff8d25d25bad80 x1876699507633280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.083991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507633280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.083996:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.083999:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762361.084001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762361.084004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762361.084014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762361.084020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffff8d25d25b8380 x1876699507633408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.084027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507633408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.084033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.084035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762361.084037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762361.084040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507633536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762361.084050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762361.084056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffff8d25d25b8700 x1876699507633536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.084063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507633536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.084068:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.084070:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762361.084071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507633536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762361.084075:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507633664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762361.084086:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762361.084094:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762361/real 1789762361] req@ffff8d25d25bb100 x1876699507633664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762361.084101:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507633664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762386 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762361.084106:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762361.084108:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762361.084110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507633664:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762361.147506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762362.171507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762363.195528:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762364.219642:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762365.243676:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762366.203494:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762366.203530:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762366.203537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762366.203555:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2343 00000100:00080000:0.0:1789762366.203571:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762366.203583:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762366.203658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507633792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762366.203721:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762366.203725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762366.203729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762366.203733:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2343 00000100:00080000:0.0:1789762366.203737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762366.203742:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762366.203754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762366.203784:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507633920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762366.203796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762366.203800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762366.203802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762366.203806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2343 00000100:00080000:0.0:1789762366.203810:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762366.203815:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762366.203821:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762366.203866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507634048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762366.203879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762366.203900:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762366.203903:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762366.203909:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2343 00000100:00080000:0.0:1789762366.203913:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762366.203916:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762366.203921:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762366.203938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507634176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762366.203949:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507633792:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762366.203966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762366.203977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762366.203987:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762366.203998:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2343 00000100:00080000:0.0:1789762366.204004:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762366.204013:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762366.204022:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762366.204047:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507634304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762366.204061:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762366.204065:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762366.204067:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762366.204072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2343 00000100:00080000:0.0:1789762366.204107:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762366.204115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762366.204117:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762366.204125:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762366.204142:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507634432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762366.204198:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffff8d25d25bb480 x1876699507633792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.204224:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507633792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.204247:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.204253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762366.204258:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507633792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762366.204268:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762366.204297:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762366.204328:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffff8d25d25b8e00 x1876699507633920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.204351:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507633920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.204382:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.204388:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762366.204393:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507633920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762366.204404:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762366.204481:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762366.204498:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffff8d25d25b9c00 x1876699507634048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.204521:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507634048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.204530:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.204533:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762366.204536:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762366.204566:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762366.204595:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762366.204606:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffff8d25d25bb100 x1876699507634176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.204618:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507634176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.204626:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.204629:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762366.204632:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762366.204638:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507634304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762366.204653:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762366.204663:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffff8d25d25b8700 x1876699507634304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.204674:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507634304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.204682:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.204686:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762366.204707:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507634304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762366.204714:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507634432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762366.204734:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762366.204745:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762366/real 1789762366] req@ffff8d25d25b8380 x1876699507634432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762366.204758:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507634432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762391 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762366.204768:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762366.204771:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762366.204774:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507634432:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762366.267559:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762367.291558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762368.315517:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762369.339608:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762370.363552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762371.323532:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762371.323549:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762371.323556:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762371.323572:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2349 00000100:00080000:2.0:1789762371.323589:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.323602:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762371.323666:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507634560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762371.323710:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762371.323716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762371.323719:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762371.323727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2349 00000100:00080000:2.0:1789762371.323735:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.323742:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762371.323758:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762371.323764:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507634560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762371.323813:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507634688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762371.323836:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789762371.323858:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762371.323860:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762371.323865:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762371.323873:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2349 00000100:00080000:2.0:1789762371.323880:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.323887:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762371.323896:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762371.323904:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffff8d25c8022300 x1876699507634560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762371.323916:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507634816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762371.323924:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507634560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762371.323963:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.323968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762371.323968:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762371.323979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507634560:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762371.323981:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762371.323986:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762371.323994:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2349 00000100:00080000:2.0:1789762371.323999:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.324005:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762371.324009:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634688:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789762371.324015:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762371.324038:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507634944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762371.324054:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762371.324062:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762371.324069:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762371.324072:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762371.324073:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffff8d25c8020380 x1876699507634688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762371.324089:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2349 00000100:00100000:1.0:1789762371.324096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507634688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762371.324097:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762371.324110:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762371.324112:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789762371.324115:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762371.324126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634688:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789762371.324129:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762371.324134:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762371.324156:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507635072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762371.324169:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762371.324173:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762371.324179:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762371.324182:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762371.324189:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffff8d25c8022d80 x1876699507634816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762371.324190:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2349 00000100:00080000:2.0:1789762371.324199:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762371.324207:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762371.324214:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762371.324219:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507634816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762371.324230:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507635200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762371.324303:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.324310:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762371.324317:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762371.324338:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762371.324364:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762371.324373:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffff8d25c8023100 x1876699507634944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762371.324425:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507634944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762371.324431:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.324440:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762371.324442:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507634944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762371.324447:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762371.324463:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762371.324471:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffff8d25c8023b80 x1876699507635072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762371.324478:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507635072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762371.324484:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.324486:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762371.324488:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762371.324492:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762371.324502:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762371.324508:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762371/real 1789762371] req@ffff8d25c8020700 x1876699507635200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762371.324515:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507635200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762396 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762371.324521:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762371.324523:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762371.324525:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635200:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762371.387519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762372.411551:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762373.435547:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762374.459597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762375.483579:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762376.443552:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762376.443561:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762376.443566:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762376.443578:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2354 00000100:00080000:1.0:1789762376.443589:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762376.443597:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762376.443657:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507635328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762376.443691:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762376.443696:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762376.443699:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762376.443707:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2354 00000100:00080000:1.0:1789762376.443712:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762376.443720:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762376.443733:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762376.443743:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507635328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762376.443793:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507635456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762376.443803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762376.443806:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762376.443808:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762376.443811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2354 00000100:00080000:1.0:1789762376.443814:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762376.443817:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762376.443822:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762376.443833:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507635584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762376.443840:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762376.443843:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762376.443844:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762376.443846:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2354 00000100:00080000:1.0:1789762376.443857:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762376.443859:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762376.443863:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762376.443871:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762376.443874:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507635712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762376.443883:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762376.443886:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762376.443888:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762376.443891:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2354 00000100:00080000:1.0:1789762376.443897:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762376.443900:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762376.443903:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762376.443913:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507635840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762376.443920:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762376.443925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762376/real 1789762376] req@ffff8d25c8168000 x1876699507635328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762376.443925:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762376.443929:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762376.443937:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2354 00000100:00080000:1.0:1789762376.443942:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762376.443948:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762376.443952:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762376.443957:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507635328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762376.443962:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507635968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762376.443983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762376.443988:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762376.443992:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507635328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762376.444016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507635456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762376.444036:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762376.444048:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762376/real 1789762376] req@ffff8d25c8169500 x1876699507635456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762376.444057:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507635456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762376.444063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762376.444065:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762376.444068:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507635456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762376.444072:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507635584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762376.444084:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762376.444093:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762376/real 1789762376] req@ffff8d25c8168700 x1876699507635584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762376.444106:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507635584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762376.444112:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762376.444113:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762376.444115:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507635584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762376.444120:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507635712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762376.444154:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762376.444164:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762376/real 1789762376] req@ffff8d25c8168380 x1876699507635712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762376.444180:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507635712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762376.444187:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762376.444189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762376.444192:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507635712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762376.444197:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762376.444213:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762376.444224:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762376/real 1789762376] req@ffff8d25c8168e00 x1876699507635840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762376.444235:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507635840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762376.444250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762376.444253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762376.444256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762376.444262:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762376.444284:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762376.444302:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762376/real 1789762376] req@ffff8d25c8169f80 x1876699507635968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762376.444319:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507635968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762401 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762376.444328:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762376.444332:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762376.444337:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507635968:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762376.507578:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762377.531537:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762378.555546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762379.579524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762380.603699:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762381.563475:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762381.563491:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762381.563496:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762381.563511:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2359 00000100:00080000:1.0:1789762381.563523:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762381.563533:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762381.563583:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507636096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762381.563615:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762381.563619:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762381.563621:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762381.563627:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2359 00000100:00080000:1.0:1789762381.563631:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762381.563638:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762381.563650:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762381.563653:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507636096:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762381.563667:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507636224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762381.563680:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762381.563683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762381.563686:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762381.563690:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2359 00000100:00080000:1.0:1789762381.563695:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762381.563699:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762381.563706:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762381.563738:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507636352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762381.563748:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762381.563751:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762381.563757:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762381.563759:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762381.563766:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2359 00000100:00080000:1.0:1789762381.563779:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762381.563784:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762381.563791:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762381.563798:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762381/real 1789762381] req@ffff8d25c816a680 x1876699507636096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762381.563809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507636480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762381.563818:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507636096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762381.563824:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762381.563830:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762381.563831:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762381.563834:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762381.563838:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762381.563843:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2359 00000100:00100000:0.0:1789762381.563844:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507636096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762381.563851:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762381.563859:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762381.563865:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762381.563875:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762381.563881:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507636608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762381.563895:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789762381.563901:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762381.563903:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762381.563907:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762381.563913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2359 00000100:00100000:0.0:1789762381.563916:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762381/real 1789762381] req@ffff8d25c816aa00 x1876699507636224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762381.563919:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762381.563929:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762381.563933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507636224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762381.563937:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762381.563945:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762381.563948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762381.563950:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762381.563956:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762381.563967:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507636736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762381.563974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762381.563986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762381/real 1789762381] req@ffff8d25c8169f80 x1876699507636352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762381.564003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507636352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762381.564012:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762381.564016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762381.564020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762381.564027:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762381.564050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762381.564061:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762381/real 1789762381] req@ffff8d25c8168e00 x1876699507636480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762381.564069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507636480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762381.564075:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762381.564077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762381.564079:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762381.564090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507636608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762381.564103:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762381.564109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762381/real 1789762381] req@ffff8d25c8168380 x1876699507636608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762381.564117:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507636608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762381.564122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762381.564124:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762381.564126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507636608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762381.564130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507636736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762381.564140:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762381.564146:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762381/real 1789762381] req@ffff8d25c8168700 x1876699507636736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762381.564159:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507636736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762406 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762381.564164:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762381.564166:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762381.564168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507636736:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762381.627531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762382.651607:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762383.675550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762384.699560:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762385.723564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762386.683526:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762386.683538:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762386.683543:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762386.683556:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2364 00000100:00080000:3.0:1789762386.683570:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762386.683578:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762386.683632:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507636864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762386.683673:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762386.683700:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762386.683704:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762386.683712:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2364 00000100:00080000:3.0:1789762386.683718:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762386.683727:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507636864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762386.683729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762386.683769:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762386.683798:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507636992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762386.683818:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762386.683819:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762386.683828:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762386.683831:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762386.683841:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2364 00000100:00080000:3.0:1789762386.683848:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762386.683855:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762386.683862:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762386/real 1789762386] req@ffff8d25d1b08700 x1876699507636864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762386.683865:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762386.683892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507636864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762386.683903:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507637120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762386.683908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762386.683917:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762386.683930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507636864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762386.683955:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762386.683961:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762386.683965:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762386.683970:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636992:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762386.683975:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2364 00000100:00080000:3.0:1789762386.683983:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762386.683991:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762386.683999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789762386.684002:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762386.684012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762386/real 1789762386] req@ffff8d25d1b0a680 x1876699507636992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762386.684033:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507636992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762386.684050:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507637248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762386.684054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762386.684061:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762386.684080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507636992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762386.684098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637120:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762386.684098:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762386.684115:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762386.684118:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762386.684125:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2364 00000100:00080000:3.0:1789762386.684131:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762386.684134:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762386.684158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762386/real 1789762386] req@ffff8d25d1b0b800 x1876699507637120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762386.684168:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762386.684175:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507637120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762386.684180:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762386.684193:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762386.684199:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762386.684212:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762386.684219:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507637376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762386.684233:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637248:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789762386.684235:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762386.684244:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762386.684247:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762386.684253:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2364 00000800:00000100:0.0:1789762386.684258:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762386.684260:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762386.684267:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762386.684274:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762386.684275:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762386/real 1789762386] req@ffff8d25d1b08e00 x1876699507637248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762386.684295:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507637504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762386.684296:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507637248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762386.684359:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762386.684366:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762386.684373:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762386.684414:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507637376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762386.684433:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762386.684441:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762386/real 1789762386] req@ffff8d25d1b09880 x1876699507637376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762386.684450:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507637376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762386.684455:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762386.684457:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762386.684460:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507637376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762386.684463:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507637504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762386.684474:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762386.684480:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762386/real 1789762386] req@ffff8d25d1b0bb80 x1876699507637504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762386.684488:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507637504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762411 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762386.684493:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762386.684494:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762386.684496:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507637504:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762386.747594:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762387.771570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762388.795574:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762389.819558:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762390.843560:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762391.803520:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762391.803532:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762391.803537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762391.803547:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2369 00000100:00080000:3.0:1789762391.803564:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762391.803571:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762391.803610:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507637632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762391.803636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762391.803650:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762391.803651:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762391.803655:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2369 00000100:00080000:3.0:1789762391.803659:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762391.803662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762391.803670:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762391.803698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507637760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762391.803710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762391.803714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762391.803717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762391.803721:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507637632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762391.803724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2369 00000100:00080000:3.0:1789762391.803731:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762391.803737:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762391.803743:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762391.803757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507637888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762391.803768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762391.803771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762391.803774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762391.803777:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2369 00000100:00080000:3.0:1789762391.803781:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762391.803785:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762391.803789:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762391.803796:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762391.803824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507638016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762391.803837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762391.803840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762391.803841:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762391.803846:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2369 00000100:00100000:0.0:1789762391.803847:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762391/real 1789762391] req@ffff8d25d1b08a80 x1876699507637632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762391.803849:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762391.803857:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762391.803864:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762391.803874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507637632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762391.803877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507638144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762391.803896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762391.803897:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762391.803903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762391.803903:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762391.803907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762391.803911:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507637632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762391.803913:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2369 00000100:00080000:3.0:1789762391.803918:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762391.803921:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762391.803924:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762391.803931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507638272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762391.803938:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762391.803959:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762391.803970:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762391/real 1789762391] req@ffff8d25d1b0aa00 x1876699507637760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762391.803979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507637760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762391.803985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762391.803987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762391.803990:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762391.803994:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762391.804005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762391.804012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762391/real 1789762391] req@ffff8d25d1b09180 x1876699507637888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762391.804025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507637888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762391.804030:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762391.804032:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762391.804034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507637888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762391.804038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762391.804049:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762391.804056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762391/real 1789762391] req@ffff8d25d1b0ad80 x1876699507638016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762391.804064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507638016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762391.804069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762391.804071:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762391.804074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762391.804080:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507638144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762391.804100:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762391.804121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762391/real 1789762391] req@ffff8d25d1b09c00 x1876699507638144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762391.804136:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507638144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762391.804146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762391.804148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762391.804166:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507638144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762391.804184:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507638272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762391.804199:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762391.804207:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762391/real 1789762391] req@ffff8d25d1b0b100 x1876699507638272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762391.804214:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507638272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762416 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762391.804220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762391.804222:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762391.804223:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507638272:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762391.867554:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762392.891614:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762393.915556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762394.939556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762395.963523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762396.923554:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762396.923569:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762396.923574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762396.923589:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2374 00000100:00080000:3.0:1789762396.923616:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762396.923628:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762396.923686:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507638400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762396.923720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762396.923724:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762396.923727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762396.923732:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2374 00000100:00080000:3.0:1789762396.923737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762396.923743:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762396.923756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762396.923764:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507638400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762396.923773:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507638528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762396.923787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762396.923792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762396.923794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762396.923799:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2374 00000100:00080000:3.0:1789762396.923804:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762396.923809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762396.923815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762396.923830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507638656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762396.923842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762396.923847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762396.923849:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762396.923853:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2374 00000100:00080000:3.0:1789762396.923857:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762396.923862:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762396.923868:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762396.923882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507638784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762396.923890:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638528:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762396.923898:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762396.923906:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762396.923907:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762396.923911:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2374 00000100:00080000:3.0:1789762396.923915:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762396.923918:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762396.923923:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762396.923935:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507638912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762396.923939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762396.923954:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762396.923958:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762396.923960:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762396.923964:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2374 00000100:00080000:3.0:1789762396.923967:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762396.923970:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762396.923973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638784:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1789762396.923977:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762396.923990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507639040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762396.923998:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507638912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762396.924030:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507639040:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762396.987557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762398.011545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762399.035526:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789762399.995747:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789762399.995788:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762399.995799:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762399.995839:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762399.995846:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762399.995855:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762399.995861:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762399.995866:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1789762399.995901:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762396/real 1789762399] req@ffff8d25d1b0b100 x1876699507639040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762399.995928:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507639040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762399.995942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762399.995947:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762399.995952:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507639040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762399.995958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762396/real 1789762399] req@ffff8d25d1b0a300 x1876699507638784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762399.995968:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507638784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762399.995974:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762399.995977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762399.995980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762399.995984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762396/real 1789762399] req@ffff8d25d1b08380 x1876699507638912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762399.995991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507638912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762399.995995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762399.995997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762399.996000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507638912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762399.996004:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762396/real 1789762399] req@ffff8d25d1b09500 x1876699507638528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762399.996012:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507638528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762399.996018:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762399.996022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762399.996026:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762399.996033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762396/real 1789762399] req@ffff8d25d1b08000 x1876699507638656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762399.996045:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507638656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762399.996053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762399.996057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762399.996060:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507638656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762399.996067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762396/real 1789762399] req@ffff8d25d1b0b480 x1876699507638400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762399.996080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507638400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762421 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762399.996088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762399.996092:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762399.996095:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507638400:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1789762400.059514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762401.083561:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762402.043708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762402.043737:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762402.043742:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762402.043751:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2379 00000100:00080000:1.0:1789762402.043761:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762402.043769:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762402.043800:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507639168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762402.043825:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762402.043828:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762402.043829:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762402.043833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2379 00000100:00080000:1.0:1789762402.043835:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762402.043838:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762402.043846:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762402.043856:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507639296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762402.043876:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762402.043880:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762402.043882:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762402.043887:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2379 00000100:00080000:1.0:1789762402.043892:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762402.043899:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762402.043909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507639168:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789762402.043909:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762402.043927:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507639424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762402.043942:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762402.043948:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762402.043950:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762402.043955:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2379 00000100:00080000:1.0:1789762402.043961:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762402.043965:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762402.043972:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762402.043988:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507639552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762402.044001:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762402.044004:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762402.044006:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762402.044011:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2379 00000100:00080000:1.0:1789762402.044017:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762402.044026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762402.044031:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762402.044039:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762402.044059:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507639680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762402.044072:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762402.044076:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762402.044078:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762402.044083:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2379 00000100:00080000:1.0:1789762402.044089:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762402.044096:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762402.044103:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762402.044115:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffff8d25c8169500 x1876699507639168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762402.044123:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507639808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762402.044184:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507639168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762402.044205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762402.044212:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762402.044218:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507639168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762402.044230:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507639296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762402.044266:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762402.044283:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffff8d25c8168000 x1876699507639296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762402.044299:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507639296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762402.044309:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762402.044314:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762402.044318:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507639296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762402.044356:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507639424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762402.044453:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762402.044482:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffff8d25c8168a80 x1876699507639424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762402.044497:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507639424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762402.044507:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762402.044512:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762402.044516:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507639424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762402.044524:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507639552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762402.044553:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762402.044588:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffff8d25c816a300 x1876699507639552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762402.044603:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507639552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762402.044619:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762402.044623:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762402.044627:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507639552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762402.044634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507639680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762402.044658:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762402.044670:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffff8d25c816ad80 x1876699507639680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762402.044683:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507639680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762402.044692:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762402.044696:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762402.044699:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507639680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762402.044709:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507639808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762402.044727:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762402.044739:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762402/real 1789762402] req@ffff8d25c8169c00 x1876699507639808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762402.044752:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507639808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762427 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762402.044762:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762402.044766:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762402.044770:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507639808:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762402.107582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762403.131512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762404.155604:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762405.179529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762406.203526:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762407.163506:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762407.163518:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762407.163522:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762407.163552:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2384 00000100:00080000:1.0:1789762407.163562:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762407.163586:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762407.163619:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507639936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762407.163643:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762407.163646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762407.163648:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762407.163652:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2384 00000100:00080000:1.0:1789762407.163655:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762407.163658:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762407.163667:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762407.163677:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507640064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762407.163686:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762407.163689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762407.163690:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762407.163694:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2384 00000100:00080000:1.0:1789762407.163697:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762407.163699:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762407.163704:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762407.163710:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507639936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762407.163717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507640192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762407.163728:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762407.163730:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762407.163732:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762407.163734:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2384 00000100:00080000:1.0:1789762407.163738:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762407.163740:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762407.163743:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762407.163752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507640320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762407.163760:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762407.163763:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762407.163764:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762407.163767:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2384 00000100:00080000:1.0:1789762407.163770:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762407.163773:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762407.163776:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762407.163782:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762407.163789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507640448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762407.163803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762407.163806:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762407.163808:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762407.163812:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2384 00000100:00080000:1.0:1789762407.163815:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762407.163818:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762407.163821:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762407.163831:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507640576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762407.163836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffff8d25c816b480 x1876699507639936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762407.163857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507639936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762407.163866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762407.163870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762407.163874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507639936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762407.163880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762407.163899:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762407.163908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffff8d25c8169880 x1876699507640064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762407.163918:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507640064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762407.163925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762407.163927:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762407.163930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762407.163950:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762407.163966:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762407.163974:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffff8d25c8169c00 x1876699507640192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762407.163983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507640192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762407.163989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762407.163991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762407.163993:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762407.163997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762407.164009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762407.164032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffff8d25c816ad80 x1876699507640320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762407.164040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507640320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762407.164046:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762407.164047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762407.164049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762407.164053:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507640448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762407.164065:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762407.164083:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffff8d25c816a300 x1876699507640448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762407.164096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507640448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762407.164101:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762407.164103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762407.164105:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507640448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762407.164108:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507640576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762407.164119:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762407.164126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762407/real 1789762407] req@ffff8d25c8168a80 x1876699507640576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762407.164139:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507640576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762432 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762407.164148:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762407.164151:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762407.164155:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507640576:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762407.227538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762408.251486:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762409.275558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762410.299584:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762411.323524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762412.283514:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762412.283525:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762412.283536:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762412.283549:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2389 00000100:00080000:2.0:1789762412.283559:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762412.283568:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762412.283602:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507640704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762412.283624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762412.283628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762412.283629:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762412.283633:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2389 00000100:00080000:2.0:1789762412.283636:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762412.283639:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762412.283648:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762412.283661:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507640832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762412.283672:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762412.283681:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507640704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762412.283681:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762412.283688:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762412.283700:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2389 00000100:00080000:2.0:1789762412.283704:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762412.283706:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762412.283717:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762412.283729:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507640960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762412.283738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762412.283741:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762412.283742:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762412.283745:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2389 00000100:00080000:2.0:1789762412.283747:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762412.283749:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762412.283752:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762412.283760:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507641088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762412.283771:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762412.283774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762412.283775:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762412.283780:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2389 00000800:00000100:0.0:1789762412.283787:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762412.283790:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762412.283798:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762412.283805:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762412.283824:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507641216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762412.283833:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762412.283835:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffff8d25c8021880 x1876699507640704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762412.283839:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762412.283847:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762412.283852:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2389 00000100:00080000:2.0:1789762412.283856:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762412.283868:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762412.283873:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762412.283878:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507640704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762412.283889:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507641344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762412.283892:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762412.283908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762412.283922:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507640704:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762412.283961:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762412.283993:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762412.284017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffff8d25c8021c00 x1876699507640832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762412.284028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507640832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762412.284036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762412.284040:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762412.284043:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762412.284054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762412.284076:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762412.284092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffff8d25c8021500 x1876699507640960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762412.284114:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507640960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762412.284124:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762412.284129:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762412.284133:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507640960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762412.284141:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762412.284160:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762412.284169:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffff8d25c8020e00 x1876699507641088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762412.284178:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507641088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762412.284184:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762412.284186:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762412.284189:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762412.284192:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507641216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762412.284203:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762412.284210:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffff8d25c8023480 x1876699507641216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762412.284226:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507641216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762412.284231:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762412.284234:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762412.284236:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507641216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762412.284239:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507641344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762412.284251:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762412.284258:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762412/real 1789762412] req@ffff8d25c8020700 x1876699507641344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762412.284266:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507641344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762437 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762412.284271:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762412.284273:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762412.284275:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507641344:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762412.347517:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762413.371533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762414.395537:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762415.419551:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762416.443528:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762417.403486:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762417.403494:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762417.403497:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762417.403506:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2395 00000100:00080000:2.0:1789762417.403520:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762417.403526:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762417.403556:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507641472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762417.403577:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762417.403580:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762417.403581:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762417.403584:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2395 00000100:00080000:2.0:1789762417.403586:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762417.403589:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762417.403596:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762417.403607:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507641600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762417.403618:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762417.403624:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762417.403625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507641472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762417.403627:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762417.403635:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2395 00000100:00080000:2.0:1789762417.403639:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762417.403644:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762417.403649:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762417.403664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507641728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762417.403674:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762417.403678:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762417.403680:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762417.403685:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762417.403689:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2395 00000100:00080000:2.0:1789762417.403695:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762417.403700:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762417.403706:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762417.403719:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507641856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762417.403725:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762417/real 1789762417] req@ffff8d25c8023b80 x1876699507641472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762417.403731:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762417.403739:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762417.403742:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762417.403747:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2395 00000100:00080000:2.0:1789762417.403751:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762417.403752:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507641472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762417.403756:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762417.403765:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762417.403769:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762417.403774:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762417.403778:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507641984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762417.403782:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507641472:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762417.403794:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762417.403797:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762417.403799:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762417.403802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2395 00000100:00080000:2.0:1789762417.403804:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762417.403806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762417.403812:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762417.403818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762417.403820:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507642112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762417.403843:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762417.403855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762417/real 1789762417] req@ffff8d25c8023100 x1876699507641600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762417.403863:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507641600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762417.403869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762417.403871:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762417.403874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762417.403877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762417.403888:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762417.403894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762417/real 1789762417] req@ffff8d25c8022d80 x1876699507641728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762417.403901:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507641728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762417.403906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762417.403908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762417.403909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762417.403913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762417.403922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762417.403928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762417/real 1789762417] req@ffff8d25c8020380 x1876699507641856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762417.403935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507641856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762417.403940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762417.403942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762417.403944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507641856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762417.403947:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507641984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762417.403956:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762417.403962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762417/real 1789762417] req@ffff8d25c8022300 x1876699507641984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762417.403968:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507641984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762417.403973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762417.403975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762417.403977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507641984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762417.403987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507642112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762417.403997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762417.404003:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762417/real 1789762417] req@ffff8d25c8020000 x1876699507642112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762417.404010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507642112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762442 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762417.404015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762417.404016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762417.404018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507642112:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762417.467521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762418.491502:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762419.515542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762420.539538:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762421.563547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762422.523549:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762422.523559:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762422.523564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762422.523576:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2400 00000100:00080000:3.0:1789762422.523590:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762422.523600:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762422.523641:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507642240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762422.523681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762422.523688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762422.523691:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762422.523698:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2400 00000100:00080000:3.0:1789762422.523704:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762422.523711:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762422.523726:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762422.523745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507642368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762422.523762:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507642240:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762422.523762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762422.523775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762422.523779:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762422.523785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2400 00000100:00080000:3.0:1789762422.523789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762422.523793:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762422.523799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762422.523815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507642496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762422.523825:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762422.523829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762422.523831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762422.523834:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2400 00000100:00080000:3.0:1789762422.523838:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762422.523841:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762422.523848:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789762422.523855:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762422.523881:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507642624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762422.523903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762422/real 1789762422] req@ffff8d25d1b09c00 x1876699507642240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762422.523918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762422.523925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789762422.523931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507642240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762422.523932:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762422.523946:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2400 00000100:00080000:3.0:1789762422.523956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762422.523961:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762422.523966:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789762422.523972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762422.523980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507642240:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789762422.523983:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762422.524009:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507642752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762422.524013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507642368:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762422.524041:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762422.524045:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762422.524048:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762422.524052:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762422.524059:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2400 00000100:00080000:3.0:1789762422.524068:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762422.524076:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762422/real 1789762422] req@ffff8d25d1b0ad80 x1876699507642368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762422.524083:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762422.524102:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762422.524107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507642368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762422.524119:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762422.524123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762422.524127:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507642368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762422.524136:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507642880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762422.524141:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507642496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762422.524175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762422.524190:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762422/real 1789762422] req@ffff8d25d1b09180 x1876699507642496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762422.524210:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507642496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762422.524218:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762422.524221:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762422.524224:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507642496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762422.524230:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507642624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762422.524246:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762422.524274:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762422/real 1789762422] req@ffff8d25d1b0aa00 x1876699507642624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762422.524289:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507642624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762422.524299:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762422.524302:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762422.524305:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507642624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762422.524328:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507642752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762422.524350:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762422.524363:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762422/real 1789762422] req@ffff8d25d1b08a80 x1876699507642752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762422.524377:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507642752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762422.524480:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762422.524484:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762422.524488:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507642752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762422.524495:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507642880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762422.524520:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762422.524533:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762422/real 1789762422] req@ffff8d25d1b0bb80 x1876699507642880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762422.524545:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507642880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762447 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762422.524553:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762422.524556:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762422.524559:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507642880:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762422.587541:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762423.611663:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762424.635544:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762425.659605:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762426.683557:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762427.643583:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762427.643597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762427.643603:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762427.643616:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2405 00000100:00080000:2.0:1789762427.643628:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762427.643638:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762427.643688:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507643008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762427.643727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762427.643731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762427.643734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762427.643739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2405 00000100:00080000:2.0:1789762427.643744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762427.643749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762427.643775:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762427.643796:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507643136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762427.643814:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507643008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762427.643815:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762427.643822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762427.643824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762427.643838:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2405 00000100:00080000:2.0:1789762427.643845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762427.643851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762427.643859:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762427.643873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507643264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762427.643884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762427.643887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762427.643889:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762427.643892:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2405 00000800:00000100:0.0:1789762427.643898:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762427.643899:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762427.643907:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762427.643912:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762427.643924:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507643392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762427.643933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762427.643937:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762427.643938:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762427.643944:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2405 00000100:00100000:0.0:1789762427.643950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762427/real 1789762427] req@ffff8d25c8023800 x1876699507643008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762427.643952:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762427.643966:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762427.643976:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762427.643979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507643008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762427.643993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789762427.644000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507643520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762427.644001:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762427.644010:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507643008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762427.644030:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762427.644035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762427.644039:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762427.644043:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643136:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762427.644051:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2405 00000100:00080000:2.0:1789762427.644058:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762427.644065:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762427.644072:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762427.644075:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762427.644087:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762427/real 1789762427] req@ffff8d25c8022a00 x1876699507643136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762427.644095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507643648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762427.644109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507643136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762427.644120:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762427.644123:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762427.644127:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762427.644134:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762427.644150:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762427.644160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762427/real 1789762427] req@ffff8d25c8022680 x1876699507643264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762427.644188:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507643264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762427.644196:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762427.644199:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762427.644202:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762427.644208:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762427.644225:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762427.644235:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762427/real 1789762427] req@ffff8d25c8021180 x1876699507643392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762427.644246:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507643392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762427.644254:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762427.644256:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762427.644259:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762427.644265:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507643520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762427.644280:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762427.644289:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762427/real 1789762427] req@ffff8d25c8020a80 x1876699507643520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762427.644300:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507643520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762427.644307:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762427.644310:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762427.644313:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507643520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762427.644345:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507643648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762427.644361:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762427.644416:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762427/real 1789762427] req@ffff8d25c8023800 x1876699507643648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762427.644466:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507643648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762452 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762427.644479:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762427.644482:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762427.644486:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507643648:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762427.707561:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762428.731590:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762429.755598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762430.779512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762431.803600:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762432.763592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762432.763609:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762432.763617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762432.763632:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2410 00000100:00080000:3.0:1789762432.763647:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762432.763656:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762432.763704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507643776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762432.763757:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762432.763762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762432.763765:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762432.763771:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2410 00000100:00080000:3.0:1789762432.763775:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762432.763781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762432.763794:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762432.763814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507643904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762432.763822:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507643776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762432.763843:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762432.763849:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762432.763852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762432.763858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2410 00000100:00080000:3.0:1789762432.763864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762432.763868:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762432.763876:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762432.763893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507644032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762432.763906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762432.763911:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762432.763913:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762432.763941:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2410 00000100:00080000:3.0:1789762432.763946:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762432.763953:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762432.763954:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762432.763968:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762432.763983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507644160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762432.764001:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762432.764007:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762432.764011:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762432.764038:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2410 00000100:00080000:3.0:1789762432.764047:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762432.764055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762432/real 1789762432] req@ffff8d25d1b09880 x1876699507643776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762432.764080:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762432.764097:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762432.764111:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507643776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762432.764129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507644288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762432.764134:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762432.764149:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762432.764162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507643776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762432.764176:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762432.764186:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762432.764191:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762432.764201:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2410 00000100:00100000:0.0:1789762432.764209:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643904:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762432.764213:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762432.764229:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762432.764239:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762432.764278:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762432.764281:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507644416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762432.764309:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762432/real 1789762432] req@ffff8d25d1b08e00 x1876699507643904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762432.764379:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507643904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762432.764423:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762432.764429:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762432.764433:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507643904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762432.764442:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762432.764473:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762432.764489:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762432/real 1789762432] req@ffff8d25d1b0b800 x1876699507644032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762432.764503:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507644032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762432.764513:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762432.764517:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762432.764521:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762432.764532:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762432.764568:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762432.764590:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762432/real 1789762432] req@ffff8d25d1b0a680 x1876699507644160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762432.764614:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507644160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762432.764629:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762432.764653:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762432.764662:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762432.764674:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507644288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762432.764728:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762432.764746:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762432/real 1789762432] req@ffff8d25d1b08700 x1876699507644288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762432.764761:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507644288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762432.764771:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762432.764775:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762432.764779:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507644288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762432.764809:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507644416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762432.764831:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762432.764844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762432/real 1789762432] req@ffff8d25d1b09880 x1876699507644416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762432.764859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507644416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762457 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762432.764868:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762432.764871:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762432.764875:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507644416:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762432.827639:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762433.851584:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762434.875635:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762435.899642:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762436.923675:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762437.883488:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762437.883497:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762437.883502:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762437.883513:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2415 00000100:00080000:3.0:1789762437.883523:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762437.883530:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762437.883565:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507644544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762437.883592:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762437.883595:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762437.883597:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762437.883601:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2415 00000100:00080000:3.0:1789762437.883605:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762437.883611:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762437.883619:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762437.883631:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507644672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762437.883645:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507644544:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762437.883649:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762437.883664:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762437.883668:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762437.883672:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2415 00000100:00080000:3.0:1789762437.883675:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762437.883679:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762437.883683:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762437.883693:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507644800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762437.883699:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762437.883703:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762437.883705:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762437.883708:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2415 00000100:00080000:3.0:1789762437.883715:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762437.883719:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762437.883722:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762437.883730:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507644928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762437.883737:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762437.883740:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762437.883742:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762437.883750:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762437.883754:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2415 00000100:00080000:3.0:1789762437.883759:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762437.883763:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762437.883768:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762437.883778:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507645056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762437.883785:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762437.883788:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762437.883790:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762437.883793:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2415 00000100:00080000:3.0:1789762437.883796:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762437.883798:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762437.883803:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762437.883816:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507645184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762437.883828:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762437/real 1789762437] req@ffff8d25d1b0bb80 x1876699507644544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762437.883854:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507644544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762437.883868:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762437.883872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762437.883876:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507644544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762437.883903:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762437.883925:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762437.883935:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762437/real 1789762437] req@ffff8d25d1b08a80 x1876699507644672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762437.883944:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507644672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762437.883950:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762437.883952:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762437.883954:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762437.883958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762437.883970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762437.883976:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762437/real 1789762437] req@ffff8d25d1b0aa00 x1876699507644800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762437.883989:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507644800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762437.883995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762437.883997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762437.883999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762437.884003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762437.884014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762437.884021:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762437/real 1789762437] req@ffff8d25d1b09180 x1876699507644928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762437.884029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507644928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762437.884034:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762437.884036:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762437.884038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507644928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762437.884041:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762437.884051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762437.884058:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762437/real 1789762437] req@ffff8d25d1b0ad80 x1876699507645056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762437.884065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507645056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762437.884075:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762437.884077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762437.884080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762437.884083:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762437.884093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762437.884099:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762437/real 1789762437] req@ffff8d25d1b09c00 x1876699507645184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762437.884106:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507645184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762462 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762437.884112:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762437.884113:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762437.884115:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645184:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762437.947522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762438.971541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762439.995596:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762441.019626:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762442.043531:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762443.003463:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762443.003471:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762443.003475:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762443.003485:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2420 00000100:00080000:3.0:1789762443.003496:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762443.003504:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762443.003542:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507645312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762443.003567:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762443.003570:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762443.003571:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762443.003574:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2420 00000100:00080000:3.0:1789762443.003589:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762443.003592:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762443.003599:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762443.003610:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507645440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762443.003620:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507645312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762443.003621:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762443.003628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762443.003630:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762443.003650:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2420 00000100:00080000:3.0:1789762443.003651:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762443.003654:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762443.003657:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762443.003665:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507645568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762443.003671:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762443.003674:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762443.003675:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762443.003678:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2420 00000100:00080000:3.0:1789762443.003680:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762443.003682:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762443.003685:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762443.003692:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507645696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762443.003696:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762443.003711:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762443.003714:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762443.003715:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762443.003719:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2420 00000100:00080000:3.0:1789762443.003722:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762443.003725:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762443.003728:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762443.003734:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffff8d25d1b0b480 x1876699507645312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762443.003737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507645824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762443.003760:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762443.003778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762443.003781:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507645312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762443.003782:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762443.003789:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2420 00000100:00080000:3.0:1789762443.003793:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762443.003798:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762443.003803:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762443.003808:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789762443.003812:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507645952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762443.003818:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762443.003826:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507645312:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762443.003853:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507645440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762443.003873:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762443.003882:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffff8d25d1b08000 x1876699507645440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762443.003891:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507645440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762443.003897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762443.003900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762443.003902:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507645440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762443.003905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507645568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762443.003916:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762443.003923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffff8d25d1b09500 x1876699507645568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762443.003930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507645568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762443.003935:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762443.003937:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762443.003939:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507645568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762443.003942:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507645696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762443.003964:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762443.003970:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffff8d25d1b08380 x1876699507645696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762443.003978:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507645696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762443.003983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762443.003984:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762443.003986:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507645696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762443.003989:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762443.003999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762443.004005:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffff8d25d1b0a300 x1876699507645824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762443.004012:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507645824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762443.004017:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762443.004018:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762443.004020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762443.004030:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762443.004039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762443.004045:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762443/real 1789762443] req@ffff8d25d1b0b100 x1876699507645952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762443.004056:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507645952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762468 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762443.004061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762443.004063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762443.004065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507645952:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762443.067536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762444.091517:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762445.115548:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762446.139489:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762447.163630:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762448.123578:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762448.123590:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762448.123596:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762448.123609:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2425 00000100:00080000:3.0:1789762448.123621:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762448.123631:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762448.123679:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507646080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762448.123717:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762448.123723:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762448.123726:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762448.123731:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2425 00000100:00080000:3.0:1789762448.123735:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762448.123739:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762448.123750:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762448.123765:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507646208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762448.123780:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762448.123784:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762448.123790:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762448.123794:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507646080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762448.123799:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2425 00000100:00080000:3.0:1789762448.123805:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762448.123809:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762448.123814:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762448.123827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507646336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762448.123838:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762448.123843:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762448.123846:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762448.123851:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2425 00000100:00080000:3.0:1789762448.123855:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762448.123859:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762448.123866:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762448.123879:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507646464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762448.123892:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762448.123893:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762448.123903:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762448.123906:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762448.123912:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2425 00000100:00080000:3.0:1789762448.123917:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762448.123922:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762448.123928:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762448.123942:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507646592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762448.123954:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762448.123961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffff8d25d1b0b100 x1876699507646080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762448.123962:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762448.123969:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762448.123984:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2425 00000100:00080000:3.0:1789762448.123994:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762448.124001:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762448.124007:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762448.124010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507646080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762448.124026:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507646720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762448.124032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762448.124047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762448.124077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507646080:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762448.124114:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762448.124178:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762448.124201:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffff8d25d1b0a300 x1876699507646208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762448.124243:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507646208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762448.124257:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762448.124262:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762448.124284:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762448.124293:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762448.124343:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762448.124366:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffff8d25d1b08380 x1876699507646336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762448.124416:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507646336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762448.124427:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762448.124432:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762448.124437:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762448.124448:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762448.124477:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762448.124493:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffff8d25d1b09500 x1876699507646464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762448.124507:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507646464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762448.124517:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762448.124520:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762448.124525:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762448.124532:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507646592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762448.124550:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762448.124562:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffff8d25d1b08000 x1876699507646592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762448.124577:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507646592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762448.124588:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762448.124608:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762448.124618:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507646592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762448.124638:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507646720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762448.124679:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762448.124702:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762448/real 1789762448] req@ffff8d25d1b0b480 x1876699507646720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762448.124725:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507646720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762473 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762448.124740:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762448.124748:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762448.124755:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507646720:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762448.187592:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762449.211631:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762450.235511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762451.259568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762452.283547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762453.243530:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762453.243542:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762453.243548:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762453.243560:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2430 00000100:00080000:2.0:1789762453.243571:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762453.243580:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762453.243628:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507646848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762453.243665:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762453.243670:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762453.243672:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762453.243677:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2430 00000100:00080000:2.0:1789762453.243682:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762453.243687:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762453.243699:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762453.243714:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507646848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762453.243725:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507646976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762453.243763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762453.243769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762453.243773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762453.243780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2430 00000100:00080000:2.0:1789762453.243787:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762453.243799:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762453.243800:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762453.243822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762453.243842:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507647104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762453.243854:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762453.243858:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762453.243861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762453.243865:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2430 00000100:00080000:2.0:1789762453.243871:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762453.243880:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffff8d25c8020000 x1876699507646848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762453.243890:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762453.243905:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762453.243921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507647232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762453.243925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507646848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762453.243957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762453.243962:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762453.243967:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507646848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762453.243986:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762453.243990:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762453.243993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762453.243998:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2430 00000100:00080000:2.0:1789762453.244011:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762453.244016:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762453.244021:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762453.244034:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762453.244039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507647360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762453.244056:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762453.244062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762453.244066:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762453.244071:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2430 00000100:00080000:2.0:1789762453.244077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762453.244085:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762453.244089:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762453.244098:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762453.244114:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffff8d25c8022300 x1876699507646976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762453.244129:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507647488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762453.244218:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507646976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762453.244245:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762453.244252:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762453.244256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507646976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762453.244264:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762453.244292:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762453.244307:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffff8d25c8020380 x1876699507647104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762453.244327:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507647104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762453.244336:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762453.244339:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762453.244342:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762453.244348:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762453.244365:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762453.244402:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffff8d25c8022d80 x1876699507647232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762453.244417:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507647232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762453.244426:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762453.244430:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762453.244433:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762453.244451:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507647360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762453.244472:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762453.244484:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffff8d25c8023100 x1876699507647360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762453.244496:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507647360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762453.244512:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762453.244515:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762453.244519:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507647360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762453.244524:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507647488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762453.244540:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762453.244550:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762453/real 1789762453] req@ffff8d25c8023b80 x1876699507647488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762453.244562:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507647488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762478 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762453.244570:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762453.244573:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762453.244576:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507647488:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762453.307617:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762454.331583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762455.355616:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762456.379521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762457.403507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762458.363576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762458.363586:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762458.363591:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762458.363600:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2436 00000100:00080000:3.0:1789762458.363610:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762458.363617:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762458.363653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507647616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762458.363682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762458.363685:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762458.363687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762458.363692:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2436 00000100:00080000:3.0:1789762458.363696:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762458.363701:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762458.363713:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762458.363740:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507647744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762458.363758:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507647616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762458.363759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762458.363772:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762458.363775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762458.363781:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2436 00000100:00080000:3.0:1789762458.363786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762458.363792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762458.363798:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762458.363813:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507647872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762458.363838:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762458.363842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1789762458.363847:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762458.363847:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762458.363856:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2436 00000100:00080000:3.0:1789762458.363863:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762458.363868:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762458.363875:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762458.363912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507648000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762458.363921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffff8d25d1b09c00 x1876699507647616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762458.363968:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762458.363972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507647616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762458.363977:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762458.363984:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762458.364000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2436 00000100:00080000:3.0:1789762458.364008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762458.364010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762458.364016:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789762458.364027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789762458.364028:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762458.364043:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507647616:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762458.364054:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507648128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762458.364068:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762458.364072:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762458.364075:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762458.364079:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2436 00000100:00080000:3.0:1789762458.364084:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762458.364090:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762458.364100:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647744:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789762458.364116:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762458.364166:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507648256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762458.364188:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762458.364215:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffff8d25d1b0ad80 x1876699507647744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762458.364234:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507647744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762458.364245:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762458.364249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762458.364254:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762458.364261:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762458.364283:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762458.364296:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffff8d25d1b09180 x1876699507647872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762458.364309:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507647872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762458.364319:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762458.364322:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762458.364325:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507647872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762458.364332:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762458.364351:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762458.364362:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffff8d25d1b0aa00 x1876699507648000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762458.364424:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507648000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762458.364432:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762458.364435:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762458.364438:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762458.364443:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507648128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762458.364463:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762458.364483:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffff8d25d1b08a80 x1876699507648128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762458.364493:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507648128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762458.364500:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762458.364502:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762458.364505:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507648128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762458.364523:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507648256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762458.364538:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762458.364546:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762458/real 1789762458] req@ffff8d25d1b0bb80 x1876699507648256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762458.364562:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507648256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762483 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762458.364568:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762458.364570:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762458.364573:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507648256:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762458.427589:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762459.451591:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762460.475634:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762461.499568:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762462.523533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762463.483579:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762463.483592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762463.483597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762463.483610:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2441 00000100:00080000:0.0:1789762463.483625:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762463.483652:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762463.483718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507648384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762463.483754:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762463.483758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762463.483760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762463.483765:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2441 00000100:00080000:0.0:1789762463.483770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762463.483774:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762463.483784:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762463.483799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507648512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762463.483816:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762463.483819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762463.483821:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762463.483825:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2441 00000100:00080000:0.0:1789762463.483828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762463.483832:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762463.483837:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762463.483848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507648640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762463.483858:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762463.483863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762463.483864:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762463.483868:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2441 00000100:00080000:0.0:1789762463.483872:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762463.483877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507648384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762463.483879:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762463.483890:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762463.483904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507648768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762463.483915:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762463.483919:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762463.483921:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762463.483926:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2441 00000100:00080000:0.0:1789762463.483946:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762463.483950:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762463.483956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762463.483967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507648896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762463.483977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762463.483981:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762463.483982:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762463.483987:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2441 00000100:00080000:0.0:1789762463.483992:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762463.483997:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762463.484003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648512:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789762463.484006:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762463.484030:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507649024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762463.484057:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762463.484098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762463.484185:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507648896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762463.484223:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507649024:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762463.547560:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762464.571536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762465.595576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789762466.555893:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789762466.555967:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762466.555984:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762466.556042:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762466.556057:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762466.556071:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762466.556120:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762466.556137:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1789762466.556168:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762463/real 1789762466] req@ffff8d25d25b9f80 x1876699507648768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762466.556213:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507648768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762466.556233:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762466.556240:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762466.556248:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762466.556264:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762463/real 1789762466] req@ffff8d25d25bb800 x1876699507648896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762466.556284:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507648896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762466.556297:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762466.556303:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762466.556309:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507648896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762466.556330:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762463/real 1789762466] req@ffff8d25d25ba680 x1876699507649024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762466.556362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507649024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762466.556417:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762466.556426:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762466.556432:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507649024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762466.556444:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762463/real 1789762466] req@ffff8d25d25bad80 x1876699507648384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762466.556465:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507648384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762466.556479:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762466.556486:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762466.556491:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507648384:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762466.556503:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762463/real 1789762466] req@ffff8d25d25bbb80 x1876699507648512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762466.556524:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507648512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762466.556539:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762466.556545:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762466.556550:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762466.556561:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762463/real 1789762466] req@ffff8d25d25b9180 x1876699507648640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762466.556582:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507648640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762488 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762466.556594:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762466.556599:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762466.556605:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507648640:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762466.619601:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762467.643558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762468.603485:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762468.603504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762468.603509:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762468.603522:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2446 00000100:00080000:0.0:1789762468.603534:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762468.603543:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762468.603584:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507649152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762468.603609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762468.603612:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762468.603613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762468.603616:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2446 00000100:00080000:0.0:1789762468.603619:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762468.603622:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762468.603631:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762468.603640:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507649280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762468.603647:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762468.603649:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762468.603650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762468.603653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2446 00000100:00080000:0.0:1789762468.603654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762468.603657:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762468.603659:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762468.603676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507649408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762468.603683:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762468.603686:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762468.603687:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762468.603690:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2446 00000100:00080000:0.0:1789762468.603692:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762468.603694:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762468.603699:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762468.603702:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507649152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762468.603710:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507649536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762468.603719:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762468.603721:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762468.603722:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762468.603725:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2446 00000100:00080000:0.0:1789762468.603727:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762468.603730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762468.603736:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762468.603748:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507649664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762468.603758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762468.603762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762468.603764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762468.603768:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2446 00000100:00080000:0.0:1789762468.603782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762468.603786:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762468.603792:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762468.603797:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762468.603810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507649792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762468.603862:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762468/real 1789762468] req@ffff8d25d25b8000 x1876699507649152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762468.603884:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507649152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762468.603896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762468.603902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762468.603907:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507649152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762468.603915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507649280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762468.603941:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762468.603957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762468/real 1789762468] req@ffff8d25d25ba300 x1876699507649280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762468.603971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507649280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762468.603980:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762468.603984:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762468.603988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507649280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762468.603995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507649408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762468.604016:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762468.604029:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762468/real 1789762468] req@ffff8d25d25b8a80 x1876699507649408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762468.604042:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507649408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762468.604051:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762468.604055:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762468.604058:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507649408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762468.604125:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507649536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762468.604151:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762468.604165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762468/real 1789762468] req@ffff8d25d25b9180 x1876699507649536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762468.604177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507649536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762468.604186:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762468.604189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762468.604192:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507649536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762468.604199:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507649664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762468.604220:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762468.604233:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762468/real 1789762468] req@ffff8d25d25bbb80 x1876699507649664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762468.604244:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507649664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762468.604252:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762468.604255:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762468.604259:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507649664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762468.604265:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507649792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762468.604285:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762468.604297:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762468/real 1789762468] req@ffff8d25d25bad80 x1876699507649792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762468.604318:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507649792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762493 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762468.604327:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762468.604330:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762468.604334:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507649792:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762468.667615:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762469.691577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762470.715603:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762471.739525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762472.763593:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762473.723464:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762473.723473:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762473.723477:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762473.723486:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2451 00000100:00080000:0.0:1789762473.723494:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762473.723500:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762473.723533:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507649920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762473.723557:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762473.723560:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762473.723561:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762473.723565:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2451 00000100:00080000:0.0:1789762473.723567:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762473.723569:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762473.723577:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762473.723585:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507650048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762473.723592:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762473.723594:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762473.723596:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762473.723598:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2451 00000100:00080000:0.0:1789762473.723599:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762473.723602:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762473.723604:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762473.723610:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507650176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762473.723617:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762473.723620:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762473.723621:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762473.723623:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2451 00000100:00080000:0.0:1789762473.723625:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762473.723628:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762473.723630:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762473.723637:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507650304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762473.723644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762473.723646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789762473.723647:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507649920:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762473.723648:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762473.723654:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2451 00000100:00080000:0.0:1789762473.723659:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762473.723662:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762473.723665:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762473.723672:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507650432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762473.723678:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762473.723680:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762473.723681:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762473.723684:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2451 00000100:00080000:0.0:1789762473.723687:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762473.723689:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762473.723692:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762473.723698:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507650560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762473.723726:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762473.723761:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762473/real 1789762473] req@ffff8d25d25ba680 x1876699507649920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762473.723774:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507649920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762473.723782:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762473.723785:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762473.723787:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507649920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762473.723792:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762473.723806:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762473.723833:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762473/real 1789762473] req@ffff8d25d25bb800 x1876699507650048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762473.723843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507650048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762473.723856:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762473.723859:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762473.723862:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762473.723866:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762473.723880:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762473.723888:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762473/real 1789762473] req@ffff8d25d25b9f80 x1876699507650176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762473.723897:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507650176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762473.723904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762473.723906:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762473.723909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762473.723944:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762473.723958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762473.723966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762473/real 1789762473] req@ffff8d25d25b8380 x1876699507650304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762473.723976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507650304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762473.723982:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762473.723985:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762473.723988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762473.723993:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507650432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762473.724016:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762473.724024:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762473/real 1789762473] req@ffff8d25d25b8700 x1876699507650432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762473.724054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507650432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762473.724061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762473.724063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762473.724066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507650432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762473.724086:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507650560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762473.724105:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762473.724113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762473/real 1789762473] req@ffff8d25d25bb100 x1876699507650560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762473.724120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507650560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762498 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762473.724126:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762473.724128:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762473.724130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507650560:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762473.787510:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762474.811539:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762475.835535:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762476.859575:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762477.883585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762478.843469:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762478.843480:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762478.843485:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762478.843495:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2456 00000100:00080000:3.0:1789762478.843506:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762478.843514:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762478.843571:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507650688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762478.843596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762478.843600:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762478.843601:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762478.843605:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2456 00000100:00080000:3.0:1789762478.843608:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762478.843611:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762478.843622:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762478.843647:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507650688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762478.843647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507650816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762478.843664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762478.843667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762478.843668:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762478.843671:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2456 00000100:00080000:3.0:1789762478.843674:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762478.843677:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762478.843680:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762478.843687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507650944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762478.843694:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762478.843709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762478.843710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762478.843713:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2456 00000100:00080000:3.0:1789762478.843717:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762478.843719:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762478.843722:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762478.843728:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507651072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762478.843736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762478.843739:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762478.843740:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762478.843742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2456 00000100:00080000:3.0:1789762478.843744:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762478.843749:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762478.843751:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762478.843757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762478.843764:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507651200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762478.843771:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762478.843775:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762478.843776:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762478.843779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2456 00000100:00080000:3.0:1789762478.843781:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762478.843783:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762478.843786:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762478.843798:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507651328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762478.843826:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762478/real 1789762478] req@ffff8d25d1b09880 x1876699507650688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762478.843849:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507650688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762478.843861:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762478.843866:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762478.843870:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507650688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762478.843912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762478.843934:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762478.843959:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762478/real 1789762478] req@ffff8d25d1b08700 x1876699507650816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762478.843972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507650816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762478.843981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762478.843984:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762478.844003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762478.844010:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762478.844029:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762478.844050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762478/real 1789762478] req@ffff8d25d1b0a680 x1876699507650944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762478.844062:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507650944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762478.844077:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762478.844081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762478.844085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507650944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762478.844092:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762478.844122:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762478.844132:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762478/real 1789762478] req@ffff8d25d1b0b800 x1876699507651072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762478.844144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507651072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762478.844153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762478.844156:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762478.844159:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762478.844165:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507651200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762478.844183:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762478.844194:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762478/real 1789762478] req@ffff8d25d1b08e00 x1876699507651200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762478.844214:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507651200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762478.844222:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762478.844225:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762478.844229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507651200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762478.844235:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507651328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762478.844253:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762478.844263:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762478/real 1789762478] req@ffff8d25d1b0bb80 x1876699507651328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762478.844274:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507651328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762503 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762478.844291:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762478.844302:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762478.844306:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507651328:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762478.907528:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762479.931570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762480.955510:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762481.979533:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762483.003495:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762483.963483:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762483.963494:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762483.963498:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762483.963510:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2461 00000100:00080000:3.0:1789762483.963520:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762483.963529:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762483.963567:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507651456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762483.963606:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762483.963610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762483.963613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762483.963617:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2461 00000100:00080000:3.0:1789762483.963621:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762483.963627:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762483.963637:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762483.963652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507651456:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762483.963654:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507651584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762483.963675:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762483.963678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762483.963680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762483.963685:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2461 00000100:00080000:3.0:1789762483.963688:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762483.963691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762483.963696:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762483.963707:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507651712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762483.963716:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762483.963719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762483.963720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762483.963723:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2461 00000100:00080000:3.0:1789762483.963729:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762483.963731:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762483.963736:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762483.963744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507651840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762483.963758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762483.963764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762483.963768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762483.963769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762483.963773:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2461 00000100:00080000:3.0:1789762483.963778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762483.963781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762483.963785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762483.963794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507651968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762483.963801:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762483.963804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762483.963806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762483.963812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2461 00000100:00100000:0.0:1789762483.963814:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762483/real 1789762483] req@ffff8d25d1b08a80 x1876699507651456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762483.963816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762483.963826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762483.963848:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507651456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762483.963864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762483.963867:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762483.963871:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507651456:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789762483.963897:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762483.963901:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762483.963911:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507652096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762483.963927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762483.963945:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762483/real 1789762483] req@ffff8d25d1b0aa00 x1876699507651584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762483.963973:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507651584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762483.963983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762483.963996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762483.964001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762483.964008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762483.964030:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762483.964042:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762483/real 1789762483] req@ffff8d25d1b09180 x1876699507651712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762483.964051:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507651712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762483.964059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762483.964062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762483.964068:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762483.964078:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762483.964096:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762483.964110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762483/real 1789762483] req@ffff8d25d1b0ad80 x1876699507651840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762483.964132:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507651840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762483.964145:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762483.964148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762483.964151:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507651840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762483.964158:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507651968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762483.964170:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762483.964177:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762483/real 1789762483] req@ffff8d25d1b09c00 x1876699507651968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762483.964183:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507651968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762483.964189:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762483.964191:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762483.964193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507651968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762483.964212:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507652096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762483.964223:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762483.964229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762483/real 1789762483] req@ffff8d25d1b0b480 x1876699507652096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762483.964238:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507652096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762508 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762483.964243:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762483.964245:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762483.964246:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507652096:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762484.027583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762485.051524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762486.075576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762487.099567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762488.123495:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762489.083481:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.083489:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762489.083492:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.083501:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2466 00000100:00080000:0.0:1789762489.083508:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.083514:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762489.083545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507652224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762489.083572:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.083574:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762489.083576:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.083579:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2466 00000100:00080000:0.0:1789762489.083581:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.083584:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762489.083591:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762489.083601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507652352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762489.083608:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.083611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762489.083612:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.083615:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2466 00000100:00080000:0.0:1789762489.083617:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.083619:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762489.083624:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507652224:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762489.083625:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762489.083635:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507652480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762489.083642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.083644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762489.083645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.083648:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2466 00000100:00080000:0.0:1789762489.083650:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.083652:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762489.083655:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762489.083665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507652608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762489.083671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.083674:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762489.083675:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.083677:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2466 00000100:00080000:0.0:1789762489.083680:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789762489.083683:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762489.083687:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762489.083692:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762489.083700:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507652736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762489.083706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762489.083709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762489.083710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762489.083713:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2466 00000100:00080000:0.0:1789762489.083716:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762489.083718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762489.083722:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffff8d25d25b9c00 x1876699507652224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762489.083723:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762489.083737:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507652864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762489.083743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507652224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762489.083758:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762489.083761:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762489.083765:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507652224:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762489.083771:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507652352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762489.083786:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762489.083795:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffff8d25d25b8e00 x1876699507652352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762489.083803:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507652352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762489.083809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762489.083811:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762489.083813:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507652352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762489.083832:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507652480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762489.083843:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762489.083849:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffff8d25d25bb480 x1876699507652480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762489.083857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507652480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762489.083862:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762489.083864:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762489.083866:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507652480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762489.083869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507652608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762489.083879:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762489.083884:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffff8d25d25bb100 x1876699507652608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762489.083892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507652608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762489.083897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762489.083899:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762489.083901:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507652608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762489.083904:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507652736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762489.083932:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762489.083940:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffff8d25d25b8700 x1876699507652736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762489.083948:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507652736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762489.083953:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762489.083955:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762489.083957:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507652736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762489.083960:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507652864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762489.083971:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762489.083977:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762489/real 1789762489] req@ffff8d25d25b8380 x1876699507652864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762489.083984:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507652864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762514 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762489.083989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762489.083991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762489.083993:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507652864:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762489.147615:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762490.171560:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762491.195508:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762492.219526:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762493.243497:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762494.203501:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762494.203513:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762494.203519:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762494.203530:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2471 00000100:00080000:2.0:1789762494.203545:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762494.203554:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762494.203597:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507652992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762494.203622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762494.203625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762494.203626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762494.203631:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2471 00000100:00080000:2.0:1789762494.203634:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762494.203638:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762494.203648:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762494.203655:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507652992:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762494.203661:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507653120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762494.203670:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762494.203673:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762494.203674:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762494.203677:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2471 00000100:00080000:2.0:1789762494.203679:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762494.203682:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762494.203685:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762494.203694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507653248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762494.203706:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762494.203709:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762494.203710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762494.203713:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2471 00000100:00080000:2.0:1789762494.203716:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762494.203718:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762494.203721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789762494.203731:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762494.203732:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507653376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762494.203743:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762494.203746:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762494.203748:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762494.203750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2471 00000100:00080000:2.0:1789762494.203753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762494.203756:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762494.203759:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762494.203767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507653504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762494.203780:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762494.203783:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffff8d25c8020700 x1876699507652992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762494.203784:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762494.203787:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762494.203794:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2471 00000100:00080000:2.0:1789762494.203797:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762494.203802:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762494.203806:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762494.203822:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507653632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762494.203832:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507652992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.203847:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.203854:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762494.203859:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507652992:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762494.203901:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762494.203928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762494.203943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffff8d25c8023480 x1876699507653120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.203956:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507653120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.203964:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.203968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762494.203972:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762494.203977:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762494.203997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762494.204010:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffff8d25c8020e00 x1876699507653248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.204021:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507653248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.204028:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.204031:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762494.204035:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762494.204040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762494.204060:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762494.204070:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffff8d25c8021500 x1876699507653376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.204082:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507653376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.204090:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.204093:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762494.204097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762494.204103:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507653504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762494.204123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762494.204135:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffff8d25c8021c00 x1876699507653504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.204147:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507653504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.204156:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.204159:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762494.204162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507653504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762494.204169:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507653632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762494.204190:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762494.204200:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762494/real 1789762494] req@ffff8d25c8021880 x1876699507653632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762494.204209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507653632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762519 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762494.204214:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762494.204217:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762494.204219:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507653632:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762494.267514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762495.291518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762496.315517:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762497.339531:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762498.363551:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762499.323490:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762499.323512:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762499.323518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762499.323531:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2477 00000100:00080000:2.0:1789762499.323544:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762499.323554:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762499.323606:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507653760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762499.323640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762499.323644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762499.323646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762499.323652:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2477 00000100:00080000:2.0:1789762499.323656:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762499.323662:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762499.323675:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762499.323694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507653888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762499.323698:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507653760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762499.323735:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762499.323740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762499.323742:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762499.323748:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2477 00000100:00080000:2.0:1789762499.323755:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762499.323760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762499.323767:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762499.323778:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762499.323793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507654016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762499.323807:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762499.323811:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762499.323813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762499.323821:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffff8d25c8021880 x1876699507653760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762499.323823:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2477 00000100:00080000:2.0:1789762499.323850:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762499.323861:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762499.323867:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507653760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762499.323869:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:1.0:1789762499.323886:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.323897:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762499.323898:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507654144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762499.323905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507653760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762499.323930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762499.323934:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762499.323937:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762499.323944:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653888:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762499.323951:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2477 00000100:00080000:2.0:1789762499.323960:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762499.323967:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762499.323974:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762499.323977:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762499.323996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffff8d25c8021c00 x1876699507653888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762499.324002:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507654272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762499.324012:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507653888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762499.324049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762499.324050:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762499.324062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762499.324063:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789762499.324068:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507653888:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762499.324070:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762499.324076:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654016:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762499.324081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2477 00000100:00080000:2.0:1789762499.324096:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762499.324102:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789762499.324108:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762499.324117:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762499.324122:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffff8d25c8021500 x1876699507654016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762499.324133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507654016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762499.324141:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507654400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762499.324145:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.324154:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762499.324165:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762499.324180:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762499.324202:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762499.324212:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffff8d25c8020e00 x1876699507654144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762499.324220:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507654144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762499.324233:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.324235:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762499.324237:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762499.324249:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507654272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762499.324261:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762499.324268:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffff8d25c8023480 x1876699507654272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762499.324276:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507654272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762499.324281:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.324283:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762499.324285:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507654272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762499.324289:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507654400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762499.324298:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762499.324304:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762499/real 1789762499] req@ffff8d25c8020700 x1876699507654400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762499.324311:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507654400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762524 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762499.324317:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762499.324319:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762499.324321:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507654400:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762499.387567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762500.411606:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762501.435522:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762502.459666:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762503.483554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762504.443512:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762504.443521:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762504.443524:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762504.443532:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2482 00000100:00080000:1.0:1789762504.443539:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762504.443545:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762504.443577:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507654528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762504.443604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762504.443607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762504.443608:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762504.443611:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2482 00000100:00080000:1.0:1789762504.443614:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762504.443616:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762504.443624:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762504.443632:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507654656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762504.443639:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762504.443641:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762504.443642:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762504.443645:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2482 00000100:00080000:1.0:1789762504.443647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762504.443650:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762504.443652:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762504.443658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507654784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762504.443664:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762504.443666:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762504.443667:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762504.443671:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2482 00000100:00080000:1.0:1789762504.443673:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762504.443676:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762504.443679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762504.443687:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507654528:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762504.443687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507654912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762504.443701:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762504.443702:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762504.443704:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762504.443706:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2482 00000100:00080000:1.0:1789762504.443708:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762504.443710:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762504.443712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762504.443720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507655040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762504.443726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762504.443728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762504.443729:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762504.443731:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2482 00000100:00080000:1.0:1789762504.443733:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762504.443735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762504.443738:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762504.443744:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507655168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762504.443770:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762504.443823:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762504/real 1789762504] req@ffff8d25c8168000 x1876699507654528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762504.443843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507654528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762504.443850:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762504.443853:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762504.443856:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507654528:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762504.443862:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762504.443878:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762504.443887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762504/real 1789762504] req@ffff8d25c8169500 x1876699507654656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762504.443895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507654656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762504.443901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762504.443903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762504.443905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762504.443908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762504.443918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762504.443925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762504/real 1789762504] req@ffff8d25c8168700 x1876699507654784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762504.443932:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507654784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762504.443942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762504.443944:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762504.443946:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762504.443965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762504.443976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762504.443983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762504/real 1789762504] req@ffff8d25c8168380 x1876699507654912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762504.443990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507654912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762504.443995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762504.443997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762504.443999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507654912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762504.444002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762504.444012:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762504.444018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762504/real 1789762504] req@ffff8d25c8168e00 x1876699507655040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762504.444025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507655040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762504.444030:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762504.444032:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762504.444034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762504.444038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762504.444047:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762504.444053:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762504/real 1789762504] req@ffff8d25c8169f80 x1876699507655168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762504.444060:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507655168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762529 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762504.444065:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762504.444067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762504.444069:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655168:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762504.507569:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762505.531544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762506.555529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762507.579600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762508.603526:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762509.563513:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762509.563525:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762509.563529:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762509.563541:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2487 00000100:00080000:3.0:1789762509.563553:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762509.563560:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762509.563605:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507655296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762509.563631:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762509.563635:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762509.563636:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762509.563640:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2487 00000100:00080000:3.0:1789762509.563642:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762509.563645:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762509.563655:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762509.563664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507655424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762509.563672:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762509.563675:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762509.563676:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762509.563679:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2487 00000100:00080000:3.0:1789762509.563682:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762509.563685:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762509.563689:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762509.563699:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507655552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762509.563705:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507655296:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762509.563709:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762509.563720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762509.563724:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762509.563729:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2487 00000100:00080000:3.0:1789762509.563731:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762509.563733:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762509.563736:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762509.563746:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507655680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762509.563753:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762509.563772:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762509.563773:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762509.563776:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2487 00000100:00080000:3.0:1789762509.563780:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762509.563784:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762509.563788:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762509.563797:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507655808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762509.563811:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762509.563820:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762509.563824:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762509.563826:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762509.563830:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2487 00000100:00080000:3.0:1789762509.563834:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762509.563837:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762509.563842:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762509.563853:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507655936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762509.563875:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762509/real 1789762509] req@ffff8d25d1b08000 x1876699507655296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762509.563896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507655296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762509.563907:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762509.563912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762509.563916:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507655296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762509.563925:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507655424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762509.563949:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762509.563962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762509/real 1789762509] req@ffff8d25d1b09500 x1876699507655424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762509.563976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507655424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762509.563984:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762509.563987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762509.563990:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507655424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762509.564019:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507655552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762509.564041:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762509.564053:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762509/real 1789762509] req@ffff8d25d1b08380 x1876699507655552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762509.564065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507655552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762509.564073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762509.564077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762509.564081:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507655552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762509.564087:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507655680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762509.564104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762509.564115:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762509/real 1789762509] req@ffff8d25d1b0a300 x1876699507655680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762509.564134:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507655680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762509.564142:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762509.564146:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762509.564150:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507655680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762509.564156:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762509.564175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762509.564186:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762509/real 1789762509] req@ffff8d25d1b0b100 x1876699507655808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762509.564197:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507655808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762509.564205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762509.564208:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762509.564212:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762509.564218:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762509.564234:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762509.564243:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762509/real 1789762509] req@ffff8d25d1b0b480 x1876699507655936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762509.564254:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507655936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762534 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762509.564267:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762509.564271:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762509.564274:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507655936:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762509.627551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762510.651526:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762511.675571:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762512.699525:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762513.723545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762514.683489:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762514.683499:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762514.683505:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762514.683515:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2492 00000100:00080000:3.0:1789762514.683525:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762514.683532:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762514.683574:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507656064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762514.683599:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762514.683602:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762514.683604:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762514.683607:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2492 00000100:00080000:3.0:1789762514.683611:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762514.683614:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762514.683624:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762514.683634:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507656192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762514.683643:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762514.683644:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507656064:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762514.683649:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762514.683652:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762514.683657:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2492 00000100:00080000:3.0:1789762514.683659:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762514.683661:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762514.683664:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762514.683673:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507656320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762514.683679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762514.683683:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762514.683686:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762514.683690:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2492 00000100:00080000:3.0:1789762514.683694:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762514.683698:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762514.683704:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762514.683717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507656448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762514.683744:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762514.683752:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762514.683757:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762514.683759:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762514.683765:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2492 00000100:00080000:3.0:1789762514.683770:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762514.683774:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762514.683780:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762514.683795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507656576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762514.683805:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762514.683810:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762514/real 1789762514] req@ffff8d25d1b09c00 x1876699507656064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762514.683811:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762514.683816:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762514.683825:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2492 00000100:00080000:3.0:1789762514.683831:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762514.683837:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762514.683843:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762514.683847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507656064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762514.683860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507656704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762514.683861:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762514.683870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762514.683878:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507656064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762514.683910:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762514.683930:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762514.683941:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762514/real 1789762514] req@ffff8d25d1b0ad80 x1876699507656192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762514.683949:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507656192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762514.683955:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762514.683958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762514.683961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762514.683965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762514.683977:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762514.683984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762514/real 1789762514] req@ffff8d25d1b09180 x1876699507656320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762514.683992:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507656320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762514.683997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762514.683999:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762514.684001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762514.684005:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762514.684016:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762514.684023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762514/real 1789762514] req@ffff8d25d1b0aa00 x1876699507656448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762514.684030:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507656448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762514.684036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762514.684037:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762514.684039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762514.684042:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507656576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762514.684053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762514.684060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762514/real 1789762514] req@ffff8d25d1b08a80 x1876699507656576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762514.684067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507656576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762514.684074:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762514.684077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762514.684080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507656576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762514.684086:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507656704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762514.684107:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762514.684119:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762514/real 1789762514] req@ffff8d25d1b0bb80 x1876699507656704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762514.684130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507656704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762539 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762514.684136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762514.684138:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762514.684140:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507656704:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762514.747545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762515.771598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762516.795543:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762517.819545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762518.843578:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762519.803594:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762519.803606:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762519.803612:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762519.803627:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2497 00000100:00080000:0.0:1789762519.803642:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762519.803654:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762519.803731:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507656832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762519.803785:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762519.803795:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762519.803799:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762519.803811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2497 00000100:00080000:0.0:1789762519.803822:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762519.803837:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762519.803839:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507656832:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762519.803855:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762519.803884:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507656960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762519.803907:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762519.803912:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762519.803915:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762519.803924:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2497 00000100:00080000:0.0:1789762519.803932:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762519.803939:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789762519.803950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762519.803952:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762519.803983:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507657088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762519.804001:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762519.804006:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762519.804010:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762519.804020:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2497 00000100:00080000:0.0:1789762519.804027:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762519.804033:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762519.804040:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762519.804042:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762519/real 1789762519] req@ffff8d25d25b9f80 x1876699507656832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762519.804063:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507657216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762519.804088:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762519.804092:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762519.804096:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762519.804105:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507656832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762519.804113:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2497 00000100:00080000:0.0:1789762519.804126:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762519.804138:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762519.804149:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789762519.804154:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762519.804164:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507657344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762519.804170:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762519.804191:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507656832:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762519.804205:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762519.804210:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762519.804214:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762519.804223:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2497 00000100:00080000:0.0:1789762519.804228:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762519.804234:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762519.804239:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762519.804245:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762519.804261:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507657472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762519.804282:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762519.804312:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762519/real 1789762519] req@ffff8d25d25bb800 x1876699507656960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762519.804324:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507656960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762519.804335:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762519.804349:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762519.804354:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507656960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762519.804360:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762519.804399:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762519.804415:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762519/real 1789762519] req@ffff8d25d25ba680 x1876699507657088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762519.804426:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507657088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762519.804437:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762519.804440:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762519.804442:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762519.804446:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762519.804461:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762519.804467:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762519/real 1789762519] req@ffff8d25d25bad80 x1876699507657216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762519.804475:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507657216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762519.804482:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762519.804485:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762519.804487:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762519.804492:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507657344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762519.804509:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762519.804524:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762519/real 1789762519] req@ffff8d25d25bbb80 x1876699507657344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762519.804540:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507657344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762519.804547:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762519.804550:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762519.804553:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507657344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762519.804572:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507657472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762519.804585:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762519.804591:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762519/real 1789762519] req@ffff8d25d25b9180 x1876699507657472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762519.804598:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507657472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762544 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762519.804604:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762519.804606:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762519.804609:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507657472:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762519.867643:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762520.891564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762521.915493:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762522.939580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762523.963605:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762524.923458:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762524.923470:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762524.923475:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762524.923487:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2502 00000100:00080000:0.0:1789762524.923495:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762524.923502:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762524.923536:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507657600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762524.923559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762524.923562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762524.923564:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762524.923567:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2502 00000100:00080000:0.0:1789762524.923570:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762524.923574:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762524.923582:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762524.923591:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507657600:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762524.923593:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507657728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762524.923605:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762524.923607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762524.923609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762524.923613:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2502 00000100:00080000:0.0:1789762524.923616:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762524.923618:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762524.923622:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762524.923631:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507657856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762524.923651:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762524.923656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762524.923658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762524.923662:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2502 00000800:00000100:2.0:1789762524.923668:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762524.923670:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762524.923693:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762524.923699:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762524.923714:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507657984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762524.923723:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762524.923727:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789762524.923727:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762524/real 1789762524] req@ffff8d25d25b8a80 x1876699507657600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762524.923730:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762524.923735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2502 00000100:00080000:0.0:1789762524.923740:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762524.923745:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762524.923751:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762524.923760:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507658112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762524.923764:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507657600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762524.923773:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762524.923781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762524.923786:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762524.923790:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762524.923793:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2502 00000100:00080000:2.0:1789762524.923798:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762524.923800:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762524.923804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507657600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762524.923809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762524.923815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762524.923826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507658240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762524.923829:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762524.923865:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762524.923877:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762524/real 1789762524] req@ffff8d25d25ba300 x1876699507657728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762524.923906:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507657728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762524.923916:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762524.923919:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762524.923923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762524.923929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762524.923948:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762524.923973:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762524/real 1789762524] req@ffff8d25d25b8000 x1876699507657856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762524.923983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507657856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762524.923989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762524.923991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762524.923994:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762524.923998:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762524.924011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762524.924018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762524/real 1789762524] req@ffff8d25d25b9180 x1876699507657984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762524.924033:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507657984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762524.924039:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762524.924041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762524.924044:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507657984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762524.924047:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507658112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762524.924059:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762524.924067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762524/real 1789762524] req@ffff8d25d25bbb80 x1876699507658112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762524.924076:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507658112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762524.924082:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762524.924084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762524.924087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507658112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762524.924107:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507658240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762524.924130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762524.924145:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762524/real 1789762524] req@ffff8d25d25bad80 x1876699507658240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762524.924161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507658240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762549 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762524.924175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762524.924178:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762524.924180:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507658240:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762524.987622:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762526.011542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762527.035531:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762528.059492:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762529.083527:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762530.043447:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.043456:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762530.043460:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.043469:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2507 00000100:00080000:2.0:1789762530.043477:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.043483:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762530.043512:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507658368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762530.043534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.043537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762530.043538:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.043541:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2507 00000100:00080000:2.0:1789762530.043543:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.043546:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762530.043553:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762530.043562:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507658496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762530.043570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.043575:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762530.043577:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507658368:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762530.043577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.043584:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2507 00000100:00080000:2.0:1789762530.043590:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.043592:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762530.043595:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762530.043616:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507658624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762530.043624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.043626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762530.043628:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.043631:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2507 00000100:00080000:2.0:1789762530.043634:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.043636:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762530.043640:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762530.043647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507658752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762530.043654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.043656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762530.043658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.043660:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2507 00000100:00080000:2.0:1789762530.043663:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.043665:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762530.043668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762530.043676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507658880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762530.043682:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762530.043684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762530.043686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762530.043689:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2507 00000100:00080000:2.0:1789762530.043691:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762530.043694:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762530.043697:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762530.043705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507659008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762530.043712:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507658496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762530.043751:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507658624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762530.043776:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507658752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762530.043798:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507658880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762530.043825:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507659008:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762530.107534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762531.131527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762532.155488:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762533.115711:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1789762533.115765:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762533.115793:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762533.115841:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762533.115856:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762533.115865:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762533.115880:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762533.115894:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789762533.115907:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762533] req@ffff8d25c8023100 x1876699507658496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762533.115954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507658496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762533.115966:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762533.115972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762533.115976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507658496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762533.115985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762533] req@ffff8d25c8022d80 x1876699507658624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762533.115999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507658624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762533.116012:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762533.116016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762533.116019:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507658624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762533.116027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762533] req@ffff8d25c8020380 x1876699507658752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762533.116040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507658752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762533.116048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762533.116051:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762533.116055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507658752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762533.116062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762533] req@ffff8d25c8022300 x1876699507658880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762533.116074:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507658880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762533.116083:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762533.116087:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762533.116090:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507658880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762533.116097:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762533] req@ffff8d25c8023b80 x1876699507658368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762533.116109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507658368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762533.116117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762533.116120:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762533.116124:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507658368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762533.116155:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762530/real 1789762533] req@ffff8d25c8020000 x1876699507659008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762533.116166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507659008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762555 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762533.116175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762533.116178:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762533.116182:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507659008:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762533.179502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762534.203475:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762535.163518:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762535.163526:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762535.163530:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762535.163538:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2512 00000100:00080000:3.0:1789762535.163546:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762535.163552:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762535.163598:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507659136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762535.163625:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762535.163629:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762535.163631:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762535.163636:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2512 00000100:00080000:3.0:1789762535.163640:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762535.163644:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762535.163655:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762535.163662:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507659136:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762535.163668:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507659264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762535.163679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762535.163682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762535.163684:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762535.163687:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2512 00000100:00080000:3.0:1789762535.163690:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762535.163693:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762535.163697:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762535.163707:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507659392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762535.163715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762535.163719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1789762535.163720:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762535.163720:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762535.163724:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2512 00000100:00080000:3.0:1789762535.163727:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762535.163730:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762535.163734:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762535.163750:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507659520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762535.163760:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762535.163762:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffff8d25d1b08e00 x1876699507659136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762535.163764:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762535.163766:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762535.163770:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2512 00000100:00080000:3.0:1789762535.163774:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762535.163778:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762535.163782:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762535.163785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507659136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762535.163791:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507659648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762535.163796:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762535.163800:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762535.163800:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762535.163804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507659136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762535.163804:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762535.163806:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762535.163809:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2512 00000100:00080000:3.0:1789762535.163813:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762535.163815:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762535.163820:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762535.163828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507659776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762535.163838:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507659264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762535.163862:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762535.163876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffff8d25d1b0b800 x1876699507659264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762535.163890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507659264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762535.163900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762535.163903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762535.163907:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507659264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762535.163914:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507659392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762535.163936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762535.163948:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffff8d25d1b0a680 x1876699507659392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762535.163959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507659392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762535.163973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762535.163977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762535.163980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507659392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762535.163986:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507659520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762535.164007:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762535.164020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffff8d25d1b08700 x1876699507659520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762535.164030:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507659520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762535.164038:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762535.164040:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762535.164044:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507659520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762535.164049:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507659648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762535.164083:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762535.164095:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffff8d25d1b09880 x1876699507659648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762535.164107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507659648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762535.164114:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762535.164117:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762535.164121:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507659648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762535.164142:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507659776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762535.164163:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762535.164174:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762535/real 1789762535] req@ffff8d25d1b0bb80 x1876699507659776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762535.164185:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507659776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762560 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762535.164193:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762535.164195:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762535.164198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507659776:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762535.227486:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762536.251542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762537.275528:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762538.299496:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762539.323496:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762540.283491:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762540.283501:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762540.283506:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762540.283518:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2517 00000100:00080000:3.0:1789762540.283541:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762540.283549:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762540.283593:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507659904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762540.283625:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762540.283629:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762540.283630:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762540.283634:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2517 00000100:00080000:3.0:1789762540.283638:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762540.283643:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762540.283655:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762540.283669:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507659904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762540.283675:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507660032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762540.283693:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762540.283696:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762540.283698:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762540.283702:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2517 00000100:00080000:3.0:1789762540.283706:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762540.283709:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762540.283714:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762540.283724:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507660160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762540.283733:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762540.283737:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762540.283739:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762540.283746:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2517 00000800:00000100:0.0:1789762540.283749:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762540.283758:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762540.283768:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762540.283776:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762540.283788:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507660288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762540.283797:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762540.283800:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762540.283802:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762540.283808:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2517 00000100:00100000:0.0:1789762540.283813:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffff8d25d1b08a80 x1876699507659904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762540.283814:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762540.283830:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762540.283834:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762540.283837:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507659904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762540.283849:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507660416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762540.283851:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762540.283869:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762540.283876:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762540.283883:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507659904:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762540.283884:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762540.283891:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762540.283898:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2517 00000100:00080000:3.0:1789762540.283901:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762540.283904:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762540.283907:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762540.283917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762540.283920:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507660544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762540.283942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762540.283960:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffff8d25d1b0aa00 x1876699507660032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762540.283978:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507660032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762540.283989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762540.283993:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762540.283997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762540.284004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762540.284026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762540.284036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffff8d25d1b09180 x1876699507660160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762540.284049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507660160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762540.284058:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762540.284062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762540.284065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762540.284072:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762540.284093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762540.284111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffff8d25d1b0ad80 x1876699507660288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762540.284131:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507660288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762540.284140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762540.284144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762540.284148:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762540.284155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507660416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762540.284176:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762540.284186:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffff8d25d1b09c00 x1876699507660416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762540.284194:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507660416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762540.284199:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762540.284202:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762540.284204:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507660416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762540.284221:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507660544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762540.284233:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762540.284240:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762540/real 1789762540] req@ffff8d25d1b0b480 x1876699507660544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762540.284247:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507660544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762565 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762540.284256:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762540.284259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762540.284261:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507660544:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762540.347644:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762541.371500:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762542.395525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762543.419496:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762544.443533:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762545.403456:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762545.403471:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762545.403477:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762545.403507:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2523 00000100:00080000:3.0:1789762545.403520:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762545.403528:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762545.403574:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507660672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762545.403604:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762545.403607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762545.403609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762545.403612:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2523 00000100:00080000:3.0:1789762545.403615:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762545.403618:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762545.403629:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762545.403639:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507660800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762545.403649:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507660672:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762545.403653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762545.403667:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762545.403669:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762545.403673:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2523 00000100:00080000:3.0:1789762545.403677:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762545.403681:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762545.403686:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762545.403696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507660928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762545.403705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762545.403708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762545.403709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762545.403712:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2523 00000100:00080000:3.0:1789762545.403714:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762545.403717:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762545.403721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762545.403728:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762545.403730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507661056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762545.403741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762545.403744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762545.403746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762545.403751:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2523 00000100:00080000:3.0:1789762545.403754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762545.403757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762545.403761:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762545.403780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507661184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762545.403782:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762545/real 1789762545] req@ffff8d25d1b0b100 x1876699507660672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762545.403846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762545.403852:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507660672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762545.403853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762545.403858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762545.403875:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2523 00000100:00080000:3.0:1789762545.403885:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762545.403894:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762545.403894:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789762545.403912:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789762545.403912:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762545.403931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507661312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762545.403934:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507660672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762545.403976:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762545.404001:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762545.404017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762545/real 1789762545] req@ffff8d25d1b0a300 x1876699507660800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762545.404026:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507660800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762545.404031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762545.404034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762545.404037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762545.404042:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762545.404053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762545.404060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762545/real 1789762545] req@ffff8d25d1b08380 x1876699507660928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762545.404067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507660928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762545.404072:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762545.404074:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762545.404076:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507660928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762545.404080:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762545.404091:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762545.404097:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762545/real 1789762545] req@ffff8d25d1b09500 x1876699507661056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762545.404105:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507661056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762545.404110:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762545.404112:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762545.404113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762545.404117:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507661184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762545.404127:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762545.404133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762545/real 1789762545] req@ffff8d25d1b08000 x1876699507661184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762545.404140:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507661184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762545.404146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762545.404147:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762545.404149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507661184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762545.404153:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507661312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762545.404163:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762545.404169:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762545/real 1789762545] req@ffff8d25d1b0b480 x1876699507661312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762545.404176:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507661312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762570 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762545.404181:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762545.404183:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762545.404185:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507661312:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762545.467564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762546.491647:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762547.515585:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762548.539539:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762549.563574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762550.523482:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762550.523492:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762550.523497:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762550.523507:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2528 00000100:00080000:3.0:1789762550.523521:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762550.523531:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762550.523569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507661440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762550.523598:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762550.523602:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762550.523604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762550.523609:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2528 00000100:00080000:3.0:1789762550.523612:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762550.523617:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762550.523626:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762550.523638:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507661568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762550.523650:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762550.523653:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507661440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762550.523655:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762550.523659:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762550.523665:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2528 00000100:00080000:3.0:1789762550.523669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762550.523673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762550.523679:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762550.523693:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507661696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762550.523703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762550.523706:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762550.523708:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762550.523714:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2528 00000800:00000100:0.0:1789762550.523719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762550.523721:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762550.523736:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762550.523742:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762550.523755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507661824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762550.523764:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762550/real 1789762550] req@ffff8d25d1b09c00 x1876699507661440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762550.523768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762550.523779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762550.523783:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762550.523793:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2528 00000100:00080000:3.0:1789762550.523798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762550.523801:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507661440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762550.523807:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762550.523816:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762550.523818:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762550.523824:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762550.523827:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507661440:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762550.523832:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507661952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762550.523844:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762550.523847:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762550.523850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762550.523854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2528 00000100:00080000:3.0:1789762550.523858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762550.523862:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661568:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762550.523865:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762550.523873:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762550.523885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762550.523890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507662080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762550.523899:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762550/real 1789762550] req@ffff8d25d1b0ad80 x1876699507661568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762550.523913:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507661568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762550.523919:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762550.523922:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762550.523926:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762550.523930:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762550.523943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762550.523950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762550/real 1789762550] req@ffff8d25d1b09180 x1876699507661696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762550.523958:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507661696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762550.523967:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762550.523969:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762550.523971:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762550.523975:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762550.523984:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762550.523990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762550/real 1789762550] req@ffff8d25d1b0aa00 x1876699507661824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762550.523997:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507661824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762550.524003:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762550.524004:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762550.524006:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507661824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762550.524009:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507661952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762550.524018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762550.524024:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762550/real 1789762550] req@ffff8d25d1b08a80 x1876699507661952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762550.524031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507661952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762550.524036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762550.524037:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762550.524040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507661952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762550.524054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507662080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762550.524064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762550.524070:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762550/real 1789762550] req@ffff8d25d1b0bb80 x1876699507662080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762550.524077:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507662080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762575 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762550.524082:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762550.524084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762550.524086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507662080:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762550.587504:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762551.611533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762552.635523:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762553.659531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762554.683502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762555.643545:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762555.643556:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762555.643560:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762555.643571:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2533 00000100:00080000:3.0:1789762555.643582:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762555.643591:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762555.643630:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507662208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762555.643659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762555.643663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762555.643664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762555.643669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2533 00000100:00080000:3.0:1789762555.643672:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762555.643675:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762555.643684:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762555.643696:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507662336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762555.643708:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762555.643711:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762555.643716:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762555.643718:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507662208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762555.643724:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2533 00000100:00080000:3.0:1789762555.643731:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762555.643735:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762555.643739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762555.643752:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507662464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762555.643761:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762555.643764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762555.643766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762555.643770:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2533 00000100:00080000:3.0:1789762555.643774:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762555.643777:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762555.643782:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762555.643796:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762555.643797:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507662592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762555.643811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762555.643814:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762555.643816:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762555.643820:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2533 00000100:00080000:3.0:1789762555.643824:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762555.643827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762555.643831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762555.643841:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507662720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762555.643851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762555.643852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762555/real 1789762555] req@ffff8d25d1b09880 x1876699507662208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762555.643860:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762555.643865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762555.643877:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2533 00000100:00080000:3.0:1789762555.643883:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762555.643887:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762555.643892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507662208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762555.643894:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762555.643915:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789762555.643919:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507662848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762555.643927:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762555.643941:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507662208:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762555.643979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507662336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762555.644019:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762555.644042:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762555/real 1789762555] req@ffff8d25d1b08700 x1876699507662336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762555.644063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507662336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762555.644076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762555.644080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762555.644085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507662336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762555.644094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507662464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762555.644135:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762555.644160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762555/real 1789762555] req@ffff8d25d1b0a680 x1876699507662464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762555.644180:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507662464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762555.644189:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762555.644193:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762555.644198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507662464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762555.644205:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507662592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762555.644232:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762555.644245:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762555/real 1789762555] req@ffff8d25d1b0b800 x1876699507662592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762555.644262:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507662592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762555.644269:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762555.644271:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762555.644274:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507662592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762555.644279:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507662720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762555.644294:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762555.644303:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762555/real 1789762555] req@ffff8d25d1b08e00 x1876699507662720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762555.644318:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507662720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762555.644325:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762555.644327:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762555.644329:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507662720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762555.644337:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507662848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762555.644353:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762555.644364:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762555/real 1789762555] req@ffff8d25d1b0bb80 x1876699507662848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762555.644409:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507662848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762580 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762555.644442:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762555.644446:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762555.644449:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507662848:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762555.707540:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762556.731600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762557.755533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762558.779574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762559.803587:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762560.763494:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762560.763505:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762560.763510:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762560.763521:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2538 00000100:00080000:0.0:1789762560.763531:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762560.763538:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762560.763581:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507662976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762560.763609:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762560.763612:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762560.763613:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762560.763618:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2538 00000100:00080000:0.0:1789762560.763620:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762560.763624:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762560.763634:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762560.763644:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507662976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762560.763647:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507663104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762560.763658:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762560.763662:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762560.763663:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762560.763667:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2538 00000100:00080000:0.0:1789762560.763669:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762560.763672:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762560.763675:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762560.763682:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507663232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762560.763688:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762560.763690:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762560.763692:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762560.763695:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2538 00000100:00080000:0.0:1789762560.763698:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762560.763701:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762560.763703:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762560.763711:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507663360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762560.763718:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762560.763721:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762560.763723:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789762560.763727:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762560.763730:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2538 00000100:00080000:0.0:1789762560.763734:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762560.763736:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762560.763740:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762560.763748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507663488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762560.763754:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762560.763756:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762560.763757:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762560.763761:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2538 00000100:00080000:0.0:1789762560.763763:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762560.763766:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762560.763769:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762560.763776:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507663616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762560.763790:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762560/real 1789762560] req@ffff8d25d25ba680 x1876699507662976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762560.763813:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507662976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762560.763823:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762560.763826:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762560.763830:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507662976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762560.763854:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762560.763873:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762560.763883:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762560/real 1789762560] req@ffff8d25d25bb800 x1876699507663104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762560.763891:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507663104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762560.763897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762560.763900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762560.763902:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762560.763905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762560.763918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762560.763925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762560/real 1789762560] req@ffff8d25d25b9f80 x1876699507663232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762560.763933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507663232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762560.763942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762560.763945:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762560.763947:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762560.763950:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762560.763961:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762560.763967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762560/real 1789762560] req@ffff8d25d25b8380 x1876699507663360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762560.763974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507663360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762560.763980:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762560.763982:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762560.763984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762560.763987:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507663488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762560.763997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762560.764003:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762560/real 1789762560] req@ffff8d25d25b8700 x1876699507663488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762560.764011:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507663488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762560.764015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762560.764017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762560.764020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507663488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762560.764024:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507663616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762560.764035:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762560.764040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762560/real 1789762560] req@ffff8d25d25bb100 x1876699507663616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762560.764048:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507663616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762585 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762560.764054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762560.764057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762560.764061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507663616:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762560.827559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762561.851680:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762562.875615:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762563.899544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762564.923568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762565.883487:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762565.883498:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762565.883503:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762565.883514:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2543 00000100:00080000:3.0:1789762565.883523:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762565.883530:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762565.883579:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507663744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762565.883602:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762565.883605:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762565.883606:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762565.883610:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2543 00000100:00080000:3.0:1789762565.883612:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762565.883615:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762565.883626:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762565.883640:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507663872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762565.883647:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507663744:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762565.883660:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762565.883665:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762565.883668:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762565.883673:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2543 00000100:00080000:3.0:1789762565.883678:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762565.883683:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762565.883688:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762565.883699:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507664000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762565.883707:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762565.883709:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762565.883710:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762565.883713:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2543 00000100:00080000:3.0:1789762565.883719:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762565.883722:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762565.883725:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762565.883733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507664128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762565.883740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789762565.883747:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762565.883747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762565.883750:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762565.883756:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2543 00000100:00080000:3.0:1789762565.883758:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762565.883761:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762565.883764:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762565.883772:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507664256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762565.883779:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762565.883782:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762565.883783:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762565.883786:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2543 00000100:00080000:3.0:1789762565.883788:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762565.883790:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762565.883793:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762565.883804:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507664384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762565.883813:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762565/real 1789762565] req@ffff8d25d1b08a80 x1876699507663744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762565.883838:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507663744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762565.883851:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762565.883856:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762565.883860:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507663744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762565.883891:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762565.883918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762565.883932:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762565/real 1789762565] req@ffff8d25d1b0aa00 x1876699507663872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762565.883945:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507663872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762565.883954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762565.883958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762565.883962:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507663872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762565.883968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762565.883987:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762565.883999:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762565/real 1789762565] req@ffff8d25d1b09180 x1876699507664000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762565.884013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507664000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762565.884022:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762565.884025:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762565.884029:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762565.884035:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762565.884055:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762565.884066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762565/real 1789762565] req@ffff8d25d1b0ad80 x1876699507664128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762565.884086:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507664128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762565.884095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762565.884099:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762565.884102:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762565.884109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507664256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762565.884129:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762565.884141:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762565/real 1789762565] req@ffff8d25d1b09c00 x1876699507664256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762565.884153:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507664256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762565.884161:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762565.884164:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762565.884168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507664256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762565.884174:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507664384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762565.884192:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762565.884203:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762565/real 1789762565] req@ffff8d25d1b0b480 x1876699507664384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762565.884215:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507664384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762590 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762565.884229:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762565.884233:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762565.884237:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507664384:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762565.947565:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762566.971599:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762567.995560:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762569.019633:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762570.043532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762571.003498:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762571.003509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762571.003513:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762571.003525:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2548 00000100:00080000:1.0:1789762571.003536:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762571.003543:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762571.003598:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507664512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762571.003623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762571.003626:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762571.003628:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762571.003631:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2548 00000100:00080000:1.0:1789762571.003634:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762571.003638:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762571.003648:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762571.003658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507664640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762571.003667:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507664512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762571.003669:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762571.003676:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762571.003677:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762571.003681:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2548 00000100:00080000:1.0:1789762571.003683:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762571.003685:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762571.003688:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762571.003698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507664768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762571.003705:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762571.003708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762571.003709:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762571.003711:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2548 00000100:00080000:1.0:1789762571.003713:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762571.003716:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762571.003719:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762571.003726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507664896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762571.003733:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762571.003736:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762571.003738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762571.003740:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2548 00000800:00000100:2.0:1789762571.003747:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762571.003753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762571.003761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762571.003766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762571.003779:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507665024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762571.003790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762571.003794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762571.003796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762571.003801:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2548 00000100:00080000:1.0:1789762571.003805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762571.003809:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762571.003813:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffff8d25c816aa00 x1876699507664512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762571.003815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762571.003830:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507665152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762571.003839:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507664512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762571.003850:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762571.003853:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762571.003858:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507664512:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762571.003882:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762571.003898:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762571.003908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffff8d25c816a680 x1876699507664640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762571.003916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507664640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762571.003922:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762571.003925:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762571.003935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762571.003939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762571.003951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762571.003959:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffff8d25c8169f80 x1876699507664768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762571.003966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507664768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762571.003972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762571.003974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762571.003976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762571.003980:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762571.003991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762571.003996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffff8d25c8168e00 x1876699507664896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762571.004004:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507664896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762571.004009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762571.004011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762571.004012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507664896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762571.004016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762571.004026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762571.004033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffff8d25c8168380 x1876699507665024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762571.004040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507665024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762571.004046:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762571.004047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762571.004049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762571.004052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762571.004062:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762571.004068:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762571/real 1789762571] req@ffff8d25c8168700 x1876699507665152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762571.004076:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507665152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762596 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762571.004081:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762571.004082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762571.004084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665152:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762571.067516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762572.091566:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762573.115524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762574.139527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762575.163602:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762576.123526:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762576.123538:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762576.123543:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762576.123555:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2553 00000100:00080000:3.0:1789762576.123567:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762576.123575:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762576.123623:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507665280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762576.123652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762576.123657:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762576.123658:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762576.123663:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2553 00000100:00080000:3.0:1789762576.123665:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762576.123668:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762576.123678:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762576.123687:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507665408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762576.123696:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762576.123699:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762576.123700:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762576.123703:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2553 00000100:00080000:3.0:1789762576.123705:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762576.123708:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762576.123711:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762576.123719:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507665536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762576.123728:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762576.123732:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762576.123737:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762576.123741:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507665280:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762576.123742:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2553 00000100:00080000:3.0:1789762576.123746:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762576.123761:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762576.123768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762576.123787:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507665664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762576.123800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762576.123804:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762576.123806:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762576.123812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2553 00000100:00080000:3.0:1789762576.123855:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762576.123862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762576.123867:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762576.123876:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762576.123882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507665792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762576.123892:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762576.123895:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762576.123896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762576.123900:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2553 00000100:00080000:3.0:1789762576.123902:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762576.123905:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762576.123910:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762576.123922:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507665920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762576.123961:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffff8d25d1b08000 x1876699507665280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762576.123994:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507665280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762576.124009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762576.124031:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762576.124036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507665280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762576.124054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507665408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762576.124084:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762576.124100:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffff8d25d1b09500 x1876699507665408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762576.124113:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507665408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762576.124123:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762576.124126:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762576.124130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507665408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762576.124180:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507665536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762576.124203:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762576.124216:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffff8d25d1b08380 x1876699507665536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762576.124229:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507665536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762576.124246:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762576.124250:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762576.124254:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507665536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762576.124304:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507665664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762576.124331:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762576.124350:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffff8d25d1b0a300 x1876699507665664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762576.124372:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507665664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762576.124441:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762576.124446:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762576.124450:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507665664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762576.124458:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762576.124488:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762576.124504:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffff8d25d1b0b100 x1876699507665792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762576.124518:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507665792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762576.124527:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762576.124531:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762576.124535:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762576.124542:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762576.124560:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762576.124571:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762576/real 1789762576] req@ffff8d25d1b0b480 x1876699507665920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762576.124584:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507665920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762601 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762576.124593:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762576.124596:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762576.124599:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507665920:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762576.187563:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762577.211520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762578.235564:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762579.259570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762580.283539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762581.243489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762581.243503:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762581.243508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762581.243521:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2558 00000100:00080000:1.0:1789762581.243534:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762581.243543:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762581.243592:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507666048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762581.243630:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762581.243634:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762581.243636:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762581.243641:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2558 00000100:00080000:1.0:1789762581.243646:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762581.243652:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762581.243665:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762581.243678:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507666048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762581.243685:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507666176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762581.243710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762581.243714:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762581.243715:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762581.243721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2558 00000100:00080000:1.0:1789762581.243725:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762581.243729:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762581.243736:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762581.243750:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507666304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762581.243760:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762581.243764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1789762581.243769:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762581.243775:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762581.243784:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2558 00000100:00080000:1.0:1789762581.243791:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762581.243796:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762581.243803:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762581.243821:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507666432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762581.243826:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffff8d25c8169500 x1876699507666048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762581.243868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762581.243874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507666048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762581.243879:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762581.243887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762581.243899:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762581.243904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2558 00000100:00080000:2.0:1789762581.243909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762581.243918:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789762581.243918:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507666048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762581.243933:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762581.243941:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762581.243948:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762581.243960:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507666560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762581.243974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762581.243980:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762581.243986:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762581.243989:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762581.243993:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffff8d25c8168000 x1876699507666176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762581.244000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2558 00000100:00080000:1.0:1789762581.244008:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762581.244023:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762581.244027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507666176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762581.244033:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789762581.244040:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762581.244046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762581.244049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762581.244056:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507666688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762581.244057:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762581.244099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762581.244116:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffff8d25c8168a80 x1876699507666304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762581.244125:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507666304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762581.244130:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762581.244133:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762581.244135:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762581.244139:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762581.244152:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762581.244158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffff8d25c816a300 x1876699507666432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762581.244170:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507666432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762581.244176:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762581.244177:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762581.244179:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762581.244192:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507666560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762581.244203:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762581.244210:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffff8d25c816ad80 x1876699507666560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762581.244217:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507666560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762581.244237:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762581.244239:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762581.244241:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507666560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762581.244246:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507666688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762581.244261:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762581.244268:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762581/real 1789762581] req@ffff8d25c8169c00 x1876699507666688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762581.244276:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507666688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762606 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762581.244285:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762581.244288:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762581.244290:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507666688:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762581.307607:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762582.331566:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762583.355551:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762584.379495:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762585.403692:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762586.363553:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762586.363566:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762586.363571:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762586.363582:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2564 00000100:00080000:2.0:1789762586.363593:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762586.363601:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762586.363649:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507666816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762586.363685:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762586.363688:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762586.363690:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762586.363694:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2564 00000100:00080000:2.0:1789762586.363697:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762586.363701:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762586.363713:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762586.363724:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507666816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762586.363726:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507666944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762586.363738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762586.363742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762586.363743:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762586.363746:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2564 00000100:00080000:2.0:1789762586.363747:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762586.363750:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762586.363754:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762586.363763:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507667072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762586.363770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762586.363772:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762586.363773:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762586.363777:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2564 00000100:00080000:2.0:1789762586.363782:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762586.363786:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762586.363792:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762586.363805:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507667200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762586.363815:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762586.363822:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762586.363827:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762586.363830:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762586.363835:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2564 00000100:00080000:2.0:1789762586.363840:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762586.363845:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762586.363850:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762586.363860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507667328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762586.363868:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762586.363870:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffff8d25c8023800 x1876699507666816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762586.363874:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762586.363878:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762586.363887:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2564 00000100:00080000:2.0:1789762586.363892:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762586.363899:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762586.363905:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762586.363915:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507666816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762586.363917:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507667456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762586.363943:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762586.363949:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762586.363953:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507666816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762586.363988:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762586.364006:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762586.364016:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffff8d25c8020a80 x1876699507666944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762586.364025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507666944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762586.364032:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762586.364034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762586.364036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507666944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762586.364041:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762586.364052:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762586.364059:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffff8d25c8021180 x1876699507667072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762586.364068:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507667072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762586.364078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762586.364080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762586.364083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762586.364086:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762586.364097:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762586.364103:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffff8d25c8022680 x1876699507667200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762586.364111:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507667200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762586.364116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762586.364118:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762586.364119:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762586.364123:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507667328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762586.364133:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762586.364139:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffff8d25c8022a00 x1876699507667328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762586.364147:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507667328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762586.364152:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762586.364154:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762586.364157:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507667328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762586.364160:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507667456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762586.364171:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762586.364177:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762586/real 1789762586] req@ffff8d25c8020000 x1876699507667456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762586.364201:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507667456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762611 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762586.364207:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762586.364209:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762586.364211:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507667456:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762586.427591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762587.451532:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762588.475578:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762589.499593:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762590.523561:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762591.483570:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762591.483610:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762591.483616:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762591.483631:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2569 00000100:00080000:2.0:1789762591.483649:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762591.483679:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762591.483741:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507667584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762591.483792:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762591.483798:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762591.483804:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762591.483812:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2569 00000100:00080000:2.0:1789762591.483821:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762591.483829:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762591.483843:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762591.483864:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507667712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762591.483881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762591.483887:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762591.483890:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762591.483896:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2569 00000100:00080000:2.0:1789762591.483900:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762591.483906:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762591.483912:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762591.483927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507667584:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762591.483933:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507667840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762591.483952:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762591.483956:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762591.483959:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762591.483964:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2569 00000100:00080000:2.0:1789762591.483968:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762591.483972:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762591.483978:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762591.483994:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507667968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762591.484008:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762591.484012:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762591.484014:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762591.484023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762591.484024:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2569 00000100:00080000:2.0:1789762591.484035:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762591.484041:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762591.484048:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762591.484063:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507668096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762591.484077:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762591.484083:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762591.484088:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762591.484096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762591/real 1789762591] req@ffff8d25c8023b80 x1876699507667584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762591.484100:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2569 00000100:00080000:2.0:1789762591.484113:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762591.484125:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762591.484136:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762591.484168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507667584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762591.484204:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507668224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762591.484207:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762591.484224:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762591.484238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507667584:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762591.484254:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762591.484301:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762591.484318:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762591/real 1789762591] req@ffff8d25c8022300 x1876699507667712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762591.484335:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507667712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762591.484345:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762591.484349:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762591.484353:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762591.484431:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762591.484460:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762591.484483:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762591/real 1789762591] req@ffff8d25c8020380 x1876699507667840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762591.484507:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507667840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762591.484523:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762591.484529:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762591.484536:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762591.484544:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762591.484564:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762591.484576:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762591/real 1789762591] req@ffff8d25c8022d80 x1876699507667968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762591.484597:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507667968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762591.484606:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762591.484610:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762591.484613:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507667968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762591.484619:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762591.484636:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762591.484647:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762591/real 1789762591] req@ffff8d25c8023100 x1876699507668096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762591.484660:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507668096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762591.484668:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762591.484671:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762591.484675:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762591.484680:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762591.484696:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762591.484707:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762591/real 1789762591] req@ffff8d25c8020700 x1876699507668224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762591.484719:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507668224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762616 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762591.484735:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762591.484739:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762591.484742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668224:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762591.547623:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762592.571550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762593.595558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762594.619570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762595.643591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762596.603480:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762596.603489:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762596.603493:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762596.603502:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2574 00000100:00080000:2.0:1789762596.603510:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762596.603516:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762596.603549:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507668352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762596.603574:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762596.603577:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762596.603579:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762596.603584:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2574 00000100:00080000:2.0:1789762596.603588:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762596.603594:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762596.603604:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762596.603621:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507668352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762596.603622:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507668480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762596.603638:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762596.603640:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762596.603642:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762596.603644:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2574 00000100:00080000:2.0:1789762596.603648:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762596.603650:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762596.603654:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762596.603664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507668608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762596.603677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762596.603682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762596.603684:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762596.603688:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2574 00000100:00080000:2.0:1789762596.603692:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762596.603697:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762596.603702:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762596.603715:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507668736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762596.603727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762596.603731:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762596.603733:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762596.603736:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2574 00000100:00100000:0.0:1789762596.603741:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507668480:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762596.603741:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762596.603748:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762596.603755:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762596.603765:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507668864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762596.603773:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762596.603776:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762596.603778:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762596.603780:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2574 00000100:00080000:2.0:1789762596.603784:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762596.603787:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762596.603791:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762596.603800:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507668608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762596.603803:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507668992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762596.603840:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507668736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762596.603872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762596.603898:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668992:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762596.667523:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762597.691585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762598.715507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789762599.675688:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1789762599.675738:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789762599.675757:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762599.675800:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762599.675810:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762599.675818:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762599.675830:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789762599.675841:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789762599.675856:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762596/real 1789762599] req@ffff8d25c8021880 x1876699507668864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762599.675889:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507668864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762599.675904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762599.675910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762599.675915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762599.675929:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762596/real 1789762599] req@ffff8d25c8020700 x1876699507668992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762599.675943:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507668992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762599.675952:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762599.675956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762599.675959:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507668992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762599.675966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762596/real 1789762599] req@ffff8d25c8020e00 x1876699507668480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762599.675980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507668480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762599.675989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762599.675993:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762599.675997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507668480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762599.676004:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762596/real 1789762599] req@ffff8d25c8021500 x1876699507668608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762599.676015:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507668608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762599.676023:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762599.676027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762599.676031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507668608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762599.676039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762596/real 1789762599] req@ffff8d25c8021c00 x1876699507668736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762599.676052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507668736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762599.676059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762599.676063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762599.676066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507668736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762599.676073:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762596/real 1789762599] req@ffff8d25c8023480 x1876699507668352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762599.676104:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507668352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762621 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762599.676113:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762599.676117:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762599.676120:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507668352:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1789762599.739501:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762600.763546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762601.723459:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762601.723468:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762601.723472:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762601.723483:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2579 00000100:00080000:2.0:1789762601.723491:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762601.723498:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762601.723528:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507669120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762601.723555:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762601.723559:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762601.723561:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762601.723566:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2579 00000100:00080000:2.0:1789762601.723569:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762601.723573:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762601.723582:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762601.723595:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507669248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762601.723605:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507669120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762601.723609:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762601.723615:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762601.723617:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762601.723621:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2579 00000100:00080000:2.0:1789762601.723626:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762601.723630:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762601.723636:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762601.723647:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507669376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762601.723657:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762601.723660:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762601.723662:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762601.723670:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2579 00000800:00000100:0.0:1789762601.723671:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762601.723675:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762601.723681:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762601.723686:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762601.723699:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507669504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762601.723708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762601.723712:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762601.723714:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762601.723720:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2579 00000100:00100000:0.0:1789762601.723721:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762601/real 1789762601] req@ffff8d25c8023100 x1876699507669120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762601.723726:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762601.723734:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762601.723743:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762601.723750:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507669120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762601.723756:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507669632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762601.723783:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762601.723788:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762601.723790:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762601.723791:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762601.723802:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762601.723802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2579 00000100:00080000:2.0:1789762601.723810:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762601.723814:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507669120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762601.723818:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762601.723825:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762601.723838:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507669760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762601.723846:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507669248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762601.723877:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762601.723892:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762601/real 1789762601] req@ffff8d25c8022d80 x1876699507669248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762601.723904:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507669248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762601.723913:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762601.723916:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762601.723919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507669248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762601.723926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507669376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762601.723947:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762601.723960:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762601/real 1789762601] req@ffff8d25c8020380 x1876699507669376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762601.723972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507669376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762601.723989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762601.723992:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762601.723996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507669376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762601.724003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507669504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762601.724027:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762601.724039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762601/real 1789762601] req@ffff8d25c8022300 x1876699507669504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762601.724051:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507669504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762601.724060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762601.724063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762601.724082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507669504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762601.724090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507669632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762601.724112:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762601.724124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762601/real 1789762601] req@ffff8d25c8023b80 x1876699507669632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762601.724137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507669632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762601.724146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762601.724149:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762601.724154:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507669632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762601.724175:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507669760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762601.724196:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762601.724208:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762601/real 1789762601] req@ffff8d25c8020000 x1876699507669760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762601.724221:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507669760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762626 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762601.724228:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762601.724231:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762601.724234:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507669760:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762601.787518:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762602.811536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762603.835524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762604.859564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762605.883542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762606.843507:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762606.843515:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762606.843518:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762606.843526:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2584 00000100:00080000:1.0:1789762606.843534:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762606.843539:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762606.843570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507669888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762606.843589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762606.843592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762606.843594:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762606.843597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2584 00000100:00080000:1.0:1789762606.843599:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762606.843602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762606.843610:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762606.843618:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507670016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762606.843626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762606.843630:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762606.843631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507669888:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762606.843632:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762606.843640:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2584 00000100:00080000:1.0:1789762606.843642:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762606.843645:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762606.843649:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762606.843658:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507670144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762606.843666:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762606.843669:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762606.843670:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762606.843672:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2584 00000100:00080000:1.0:1789762606.843686:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762606.843688:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762606.843691:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762606.843697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762606.843702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507670272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762606.843710:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762606.843712:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762606.843714:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762606.843717:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2584 00000100:00080000:1.0:1789762606.843722:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762606.843725:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762606.843727:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762606.843734:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507670400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762606.843741:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762606.843743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762606.843747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762606.843748:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762606/real 1789762606] req@ffff8d25c8169880 x1876699507669888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762606.843752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2584 00000100:00080000:1.0:1789762606.843757:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762606.843764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762606.843770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762606.843778:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507670528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762606.843784:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507669888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762606.843798:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762606.843803:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762606.843808:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507669888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762606.843830:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762606.843853:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762606.843865:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762606/real 1789762606] req@ffff8d25c816b480 x1876699507670016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762606.843877:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507670016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762606.843892:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762606.843896:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762606.843899:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762606.843905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762606.843922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762606.843932:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762606/real 1789762606] req@ffff8d25c8169c00 x1876699507670144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762606.843945:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507670144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762606.843954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762606.843957:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762606.843961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762606.843967:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762606.843984:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762606.843996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762606/real 1789762606] req@ffff8d25c816ad80 x1876699507670272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762606.844008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507670272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762606.844016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762606.844019:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762606.844024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762606.844044:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507670400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762606.844064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762606.844076:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762606/real 1789762606] req@ffff8d25c816a300 x1876699507670400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762606.844087:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507670400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762606.844094:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762606.844097:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762606.844100:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507670400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762606.844107:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507670528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762606.844126:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762606.844138:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762606/real 1789762606] req@ffff8d25c8168a80 x1876699507670528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762606.844150:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507670528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762631 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762606.844158:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762606.844161:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762606.844165:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507670528:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762606.907569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762607.931521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762608.955515:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762609.979581:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762611.003554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762611.963475:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762611.963485:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762611.963490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762611.963499:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2589 00000100:00080000:3.0:1789762611.963509:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762611.963515:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762611.963550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507670656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762611.963575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762611.963579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762611.963582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762611.963587:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2589 00000100:00080000:3.0:1789762611.963591:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762611.963596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762611.963605:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762611.963607:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507670656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762611.963621:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507670784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762611.963634:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762611.963637:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762611.963640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762611.963644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2589 00000100:00080000:3.0:1789762611.963648:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762611.963652:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762611.963657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762611.963667:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762611.963672:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507670912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762611.963684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762611.963687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762611.963690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762611.963694:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2589 00000100:00080000:3.0:1789762611.963698:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762611.963702:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762611.963706:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762611/real 1789762611] req@ffff8d25d1b09c00 x1876699507670656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789762611.963708:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762611.963726:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507671040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762611.963728:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507670656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762611.963756:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762611.963759:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762611.963760:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762611.963767:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507670656:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762611.963767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762611.963772:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762611.963778:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2589 00000100:00080000:3.0:1789762611.963782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762611.963785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762611.963790:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762611.963794:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762611.963804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507671168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762611.963815:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762611.963821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762611.963825:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762611.963828:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762611.963829:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762611/real 1789762611] req@ffff8d25d1b0ad80 x1876699507670784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762611.963835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2589 00000100:00080000:3.0:1789762611.963842:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762611.963846:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507670784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762611.963850:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789762611.963860:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789762611.963860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762611.963866:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762611.963870:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762611.963878:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762611.963879:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507671296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762611.963894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762611.963905:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762611/real 1789762611] req@ffff8d25d1b09180 x1876699507670912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762611.963917:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507670912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762611.963923:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762611.963925:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762611.963927:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507670912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762611.963930:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762611.963941:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762611.963947:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762611/real 1789762611] req@ffff8d25d1b0aa00 x1876699507671040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762611.963954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507671040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762611.963959:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762611.963961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762611.963963:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762611.963975:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507671168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762611.963986:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762611.964011:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762611/real 1789762611] req@ffff8d25d1b08a80 x1876699507671168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762611.964020:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507671168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762611.964029:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762611.964031:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762611.964033:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507671168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762611.964037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507671296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762611.964051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762611.964058:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762611/real 1789762611] req@ffff8d25d1b0bb80 x1876699507671296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762611.964065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507671296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762636 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762611.964070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762611.964072:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762611.964073:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507671296:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762612.027505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762613.051522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762614.075553:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762615.099506:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762616.123548:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762617.083448:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762617.083458:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762617.083463:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762617.083474:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2594 00000100:00080000:3.0:1789762617.083483:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762617.083490:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762617.083520:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507671424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762617.083541:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762617.083544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762617.083546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762617.083549:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2594 00000100:00080000:3.0:1789762617.083551:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762617.083554:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762617.083561:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762617.083569:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507671552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762617.083577:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762617.083579:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762617.083581:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762617.083583:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2594 00000100:00080000:3.0:1789762617.083585:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762617.083590:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507671424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762617.083590:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762617.083595:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762617.083605:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507671680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762617.083616:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762617.083618:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762617.083620:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762617.083622:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2594 00000100:00080000:3.0:1789762617.083624:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762617.083626:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762617.083630:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762617.083637:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507671808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762617.083644:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762617.083646:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762617.083647:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762617.083649:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2594 00000100:00080000:3.0:1789762617.083652:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789762617.083655:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762617.083658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762617.083663:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762617.083671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507671936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762617.083677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762617.083680:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762617.083681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762617.083684:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2594 00000100:00080000:3.0:1789762617.083687:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762617.083690:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762617.083693:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762617.083695:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762617] req@ffff8d25d1b08e00 x1876699507671424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762617.083705:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507672064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762617.083725:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507671424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762617.083734:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762617.083738:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762617.083741:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507671424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762617.083746:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762617.083762:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762617.083770:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762617] req@ffff8d25d1b0b800 x1876699507671552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762617.083778:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507671552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762617.083783:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762617.083786:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762617.083788:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762617.083808:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762617.083819:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762617.083825:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762617] req@ffff8d25d1b0a680 x1876699507671680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762617.083833:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507671680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762617.083838:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762617.083839:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762617.083841:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762617.083845:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762617.083855:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762617.083861:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762617] req@ffff8d25d1b08700 x1876699507671808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762617.083868:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507671808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762617.083873:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762617.083875:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762617.083876:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507671808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762617.083880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507671936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762617.083889:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762617.083894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762617] req@ffff8d25d1b09880 x1876699507671936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762617.083902:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507671936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762617.083906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762617.083908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762617.083910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507671936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762617.083913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507672064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762617.083922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762617.083928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762617/real 1789762617] req@ffff8d25d1b0bb80 x1876699507672064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762617.083939:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507672064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762642 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762617.083944:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762617.083946:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762617.083948:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507672064:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762617.147499:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762618.171577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762619.195568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762620.219558:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762621.243505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762622.203508:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762622.203518:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762622.203522:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762622.203534:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2599 00000100:00080000:1.0:1789762622.203546:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762622.203554:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762622.203592:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507672192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762622.203623:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762622.203627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762622.203629:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762622.203634:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2599 00000100:00080000:1.0:1789762622.203639:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762622.203644:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762622.203656:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762622.203671:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507672320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762622.203684:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507672192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762622.203712:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762622.203716:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762622.203718:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762622.203723:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2599 00000100:00080000:1.0:1789762622.203726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762622.203730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762622.203735:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762622.203747:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507672448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762622.203758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762622.203762:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762622.203764:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762622.203768:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2599 00000100:00080000:1.0:1789762622.203783:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762622.203788:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762622.203794:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762622.203795:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762622.203815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507672576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762622.203828:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762622.203831:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762622.203833:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762622.203837:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2599 00000100:00080000:1.0:1789762622.203841:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762622.203846:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762622.203851:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762622.203861:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffff8d25c8168000 x1876699507672192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762622.203866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507672704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762622.203945:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507672192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762622.203950:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762622.203975:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762622.203979:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762622.203980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762622.203999:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762622.204000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2599 00000100:00100000:0.0:1789762622.204013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507672192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762622.204014:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762622.204022:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762622.204031:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762622.204048:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507672832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762622.204061:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507672320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762622.204104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762622.204119:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffff8d25c8169500 x1876699507672320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762622.204128:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507672320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762622.204141:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762622.204144:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762622.204146:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507672320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762622.204151:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507672448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762622.204164:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762622.204171:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffff8d25c8168700 x1876699507672448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762622.204179:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507672448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762622.204184:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762622.204186:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762622.204188:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507672448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762622.204193:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507672576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762622.204205:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762622.204212:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffff8d25c8168380 x1876699507672576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762622.204219:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507672576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762622.204225:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762622.204227:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762622.204229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507672576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762622.204233:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507672704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762622.204245:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762622.204251:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffff8d25c8168e00 x1876699507672704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762622.204258:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507672704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762622.204263:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762622.204265:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762622.204267:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507672704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762622.204283:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507672832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762622.204295:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762622.204301:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762622/real 1789762622] req@ffff8d25c8169f80 x1876699507672832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762622.204309:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507672832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762647 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762622.204314:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762622.204316:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762622.204317:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507672832:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762622.267565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762623.291493:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762624.315558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762625.339519:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762626.363577:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762627.323534:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762627.323543:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762627.323546:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762627.323555:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2605 00000100:00080000:1.0:1789762627.323567:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762627.323574:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762627.323612:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507672960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762627.323640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762627.323644:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762627.323645:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762627.323650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2605 00000100:00080000:1.0:1789762627.323654:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762627.323661:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762627.323671:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762627.323691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507673088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762627.323692:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507672960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762627.323730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762627.323734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762627.323736:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762627.323742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2605 00000100:00080000:1.0:1789762627.323745:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762627.323749:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762627.323754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762627.323768:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507673216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762627.323776:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762627.323779:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762627.323780:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762627.323783:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2605 00000100:00080000:1.0:1789762627.323786:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762627.323788:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762627.323792:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762627.323795:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762627.323808:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507673344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762627.323817:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762627.323819:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762627.323820:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762627.323823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2605 00000100:00080000:1.0:1789762627.323825:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762627.323827:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762627.323830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762627.323836:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507673472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762627.323842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762627.323845:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762627.323846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762627.323849:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2605 00000100:00080000:1.0:1789762627.323851:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762627.323854:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762627.323858:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762627.323864:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507673600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762627.324107:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffff8d25c816a680 x1876699507672960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762627.324137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507672960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762627.324152:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762627.324158:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762627.324162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507672960:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762627.324189:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762627.324211:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762627.324221:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffff8d25c816aa00 x1876699507673088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762627.324229:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507673088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762627.324241:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762627.324243:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762627.324245:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762627.324249:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762627.324261:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762627.324267:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffff8d25c8169f80 x1876699507673216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762627.324275:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507673216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762627.324280:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762627.324282:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762627.324284:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762627.324288:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762627.324299:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762627.324306:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffff8d25c8168e00 x1876699507673344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762627.324314:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507673344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762627.324319:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762627.324321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762627.324323:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762627.324327:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507673472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762627.324339:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762627.324345:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffff8d25c8168380 x1876699507673472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762627.324352:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507673472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762627.324357:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762627.324359:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762627.324361:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507673472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762627.324365:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507673600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762627.324411:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762627.324430:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762627/real 1789762627] req@ffff8d25c8168700 x1876699507673600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762627.324446:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507673600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762652 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762627.324452:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762627.324455:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762627.324457:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507673600:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762627.387563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762628.411583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762629.435514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762630.459576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762631.483556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762632.443523:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762632.443537:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762632.443542:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762632.443553:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2610 00000100:00080000:2.0:1789762632.443565:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762632.443574:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762632.443618:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507673728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762632.443645:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762632.443647:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762632.443649:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762632.443653:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2610 00000100:00080000:2.0:1789762632.443655:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762632.443659:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762632.443669:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762632.443681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507673856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762632.443691:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762632.443691:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507673728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762632.443708:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762632.443711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762632.443716:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2610 00000100:00080000:2.0:1789762632.443718:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762632.443721:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762632.443724:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762632.443731:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507673984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762632.443738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762632.443741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762632.443743:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762632.443745:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2610 00000100:00080000:2.0:1789762632.443748:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762632.443750:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762632.443754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762632.443761:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507674112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762632.443768:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762632.443770:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762632.443771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762632.443774:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2610 00000100:00080000:2.0:1789762632.443778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762632.443781:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762632.443785:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762632.443793:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507674240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762632.443800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762632.443805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:3.0:1789762632.443805:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762632.443809:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762632.443813:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2610 00000100:00080000:2.0:1789762632.443815:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762632.443818:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762632.443822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762632.443831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507674368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762632.443884:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762632/real 1789762632] req@ffff8d25c8022a00 x1876699507673728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762632.443917:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507673728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762632.443931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762632.443936:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762632.443940:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507673728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762632.443972:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762632.444012:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762632.444028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762632/real 1789762632] req@ffff8d25c8022680 x1876699507673856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762632.444042:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507673856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762632.444052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762632.444056:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762632.444060:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762632.444067:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762632.444088:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762632.444108:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762632/real 1789762632] req@ffff8d25c8021180 x1876699507673984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762632.444129:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507673984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762632.444139:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762632.444142:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762632.444146:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507673984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762632.444152:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762632.444184:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762632.444196:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762632/real 1789762632] req@ffff8d25c8020a80 x1876699507674112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762632.444209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507674112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762632.444217:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762632.444221:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762632.444225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762632.444231:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507674240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762632.444251:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762632.444261:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762632/real 1789762632] req@ffff8d25c8023800 x1876699507674240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762632.444273:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507674240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762632.444309:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762632.444313:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762632.444317:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507674240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762632.444323:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507674368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762632.444343:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762632.444354:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762632/real 1789762632] req@ffff8d25c8020000 x1876699507674368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762632.444367:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507674368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762657 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762632.444405:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762632.444408:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762632.444412:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507674368:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762632.507567:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762633.531519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762634.555536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762635.579534:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762636.603532:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762637.563474:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762637.563485:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762637.563490:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762637.563502:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2615 00000100:00080000:0.0:1789762637.563512:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762637.563520:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762637.563563:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507674496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762637.563589:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762637.563592:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762637.563594:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762637.563597:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2615 00000100:00080000:0.0:1789762637.563599:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762637.563602:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762637.563611:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762637.563620:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507674624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762637.563628:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762637.563631:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762637.563632:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762637.563635:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2615 00000100:00080000:0.0:1789762637.563638:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762637.563642:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762637.563647:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762637.563653:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507674496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762637.563656:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507674752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762637.563667:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762637.563670:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762637.563671:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762637.563673:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2615 00000100:00080000:0.0:1789762637.563675:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762637.563677:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762637.563681:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762637.563690:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507674880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762637.563698:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762637.563701:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762637.563702:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762637.563704:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2615 00000100:00080000:0.0:1789762637.563707:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762637.563709:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762637.563712:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762637.563719:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507675008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762637.563728:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762637.563733:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762637.563735:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762637.563738:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762637.563743:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2615 00000100:00080000:0.0:1789762637.563749:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762637.563753:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762637.563760:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762637.563773:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507675136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762637.563815:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762637/real 1789762637] req@ffff8d25d25bb480 x1876699507674496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762637.563836:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507674496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762637.563846:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762637.563851:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762637.563855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507674496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762637.563863:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762637.563884:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762637.563894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762637/real 1789762637] req@ffff8d25d25b8e00 x1876699507674624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762637.563903:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507674624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762637.563909:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762637.563911:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762637.563913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762637.563938:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762637.563950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762637.563957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762637/real 1789762637] req@ffff8d25d25b9c00 x1876699507674752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762637.563964:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507674752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762637.563970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762637.563972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762637.563975:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762637.563978:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762637.563990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762637.563996:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762637/real 1789762637] req@ffff8d25d25bb100 x1876699507674880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762637.564004:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507674880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762637.564009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762637.564011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762637.564012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507674880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762637.564016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762637.564026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762637.564032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762637/real 1789762637] req@ffff8d25d25b8700 x1876699507675008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762637.564040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507675008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762637.564045:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762637.564047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762637.564049:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762637.564052:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762637.564061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762637.564067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762637/real 1789762637] req@ffff8d25d25b8380 x1876699507675136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762637.564080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507675136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762662 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762637.564085:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762637.564087:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762637.564089:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675136:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762637.627511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762638.651526:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762639.675592:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762640.699541:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762641.723524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762642.683477:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762642.683488:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762642.683493:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762642.683506:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2620 00000100:00080000:0.0:1789762642.683518:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762642.683526:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762642.683584:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507675264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762642.683618:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762642.683621:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762642.683623:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762642.683627:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2620 00000100:00080000:0.0:1789762642.683629:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762642.683632:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762642.683643:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762642.683653:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507675392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762642.683673:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762642.683677:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762642.683678:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762642.683683:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2620 00000100:00080000:0.0:1789762642.683687:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762642.683692:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762642.683697:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762642.683710:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507675520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762642.683722:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762642.683725:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762642.683727:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762642.683731:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2620 00000100:00080000:0.0:1789762642.683737:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762642.683741:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762642.683746:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762642.683757:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507675648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762642.683782:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762642.683785:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762642.683787:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762642.683805:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2620 00000100:00080000:0.0:1789762642.683808:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762642.683814:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762642.683819:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762642.683825:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507675264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762642.683835:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507675776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762642.683846:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762642.683849:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762642.683850:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762642.683852:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2620 00000100:00080000:0.0:1789762642.683865:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762642.683867:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762642.683871:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762642.683882:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507675904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762642.683950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762642.684018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762642/real 1789762642] req@ffff8d25d25b9f80 x1876699507675264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762642.684047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507675264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762642.684066:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762642.684073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762642.684080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507675264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762642.684109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507675392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762642.684156:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762642.684182:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762642/real 1789762642] req@ffff8d25d25bb800 x1876699507675392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762642.684199:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507675392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762642.684219:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762642.684223:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762642.684228:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507675392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762642.684252:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507675520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762642.684273:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762642.684284:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762642/real 1789762642] req@ffff8d25d25ba680 x1876699507675520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762642.684295:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507675520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762642.684302:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762642.684305:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762642.684307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507675520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762642.684337:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507675648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762642.684374:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762642.684426:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762642/real 1789762642] req@ffff8d25d25bad80 x1876699507675648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762642.684437:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507675648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762642.684445:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762642.684448:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762642.684452:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507675648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762642.684458:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762642.684477:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762642.684486:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762642/real 1789762642] req@ffff8d25d25bbb80 x1876699507675776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762642.684497:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507675776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762642.684504:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762642.684507:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762642.684510:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762642.684515:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762642.684530:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762642.684539:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762642/real 1789762642] req@ffff8d25d25b9180 x1876699507675904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762642.684550:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507675904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762667 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762642.684557:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762642.684559:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762642.684562:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507675904:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762642.747545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762643.771576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762644.795528:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762645.819613:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762646.843616:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762647.803563:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762647.803576:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762647.803581:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762647.803594:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2625 00000100:00080000:3.0:1789762647.803606:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762647.803616:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762647.803682:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507676032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762647.803720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762647.803751:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762647.803755:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762647.803768:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2625 00000100:00080000:3.0:1789762647.803774:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762647.803780:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762647.803793:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762647.803815:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507676032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762647.803819:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507676160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762647.803843:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762647.803848:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762647.803851:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762647.803858:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2625 00000100:00080000:3.0:1789762647.803862:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762647.803867:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762647.803874:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762647.803890:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507676288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762647.803903:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762647.803908:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:0.0:1789762647.803915:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762647.803916:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762647.803929:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2625 00000100:00080000:3.0:1789762647.803937:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762647.803942:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762647.803950:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762647.803967:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507676416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762647.803981:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762647.803987:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762647/real 1789762647] req@ffff8d25d1b08a80 x1876699507676032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762647.803989:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762647.803996:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762647.804010:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2625 00000100:00080000:3.0:1789762647.804019:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762647.804032:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762647.804039:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762647.804054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507676032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762647.804057:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507676544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762647.804097:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762647.804108:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762647.804111:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762647.804117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762647.804119:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2625 00000100:00080000:3.0:1789762647.804129:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762647.804139:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762647.804140:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762647.804155:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762647.804163:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507676032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762647.804181:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507676672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762647.804219:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762647.804271:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762647.804300:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762647/real 1789762647] req@ffff8d25d1b0aa00 x1876699507676160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762647.804325:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507676160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762647.804337:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762647.804341:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762647.804345:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762647.804354:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762647.804419:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762647.804436:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762647/real 1789762647] req@ffff8d25d1b09180 x1876699507676288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762647.804451:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507676288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762647.804462:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762647.804466:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762647.804470:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762647.804478:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762647.804500:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762647.804513:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762647/real 1789762647] req@ffff8d25d1b0ad80 x1876699507676416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762647.804527:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507676416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762647.804537:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762647.804540:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762647.804544:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762647.804550:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507676544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762647.804569:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762647.804580:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762647/real 1789762647] req@ffff8d25d1b09c00 x1876699507676544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762647.804594:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507676544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762647.804604:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762647.804607:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762647.804611:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507676544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762647.804617:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507676672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762647.804634:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762647.804646:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762647/real 1789762647] req@ffff8d25d1b0b480 x1876699507676672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762647.804660:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507676672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762672 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762647.804670:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762647.804673:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762647.804677:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507676672:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762647.867637:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762648.891492:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762649.915522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762650.939584:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762651.963646:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762652.923475:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762652.923483:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762652.923487:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762652.923494:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2630 00000100:00080000:0.0:1789762652.923502:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762652.923507:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762652.923539:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507676800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762652.923560:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762652.923563:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762652.923564:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762652.923568:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2630 00000100:00080000:0.0:1789762652.923570:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762652.923573:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762652.923581:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762652.923589:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507676800:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762652.923591:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507676928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762652.923601:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762652.923604:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762652.923605:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762652.923608:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2630 00000100:00080000:0.0:1789762652.923610:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762652.923612:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762652.923615:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762652.923623:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507677056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762652.923629:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762652.923632:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762652.923633:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762652.923636:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2630 00000100:00080000:0.0:1789762652.923638:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762652.923640:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762652.923643:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762652.923652:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507677184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762652.923659:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762652.923662:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762652.923665:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762652.923666:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762652.923670:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2630 00000100:00080000:0.0:1789762652.923673:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762652.923676:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762652.923679:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762652.923700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507677312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762652.923707:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762652.923709:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762652.923711:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762652.923714:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2630 00000100:00080000:0.0:1789762652.923717:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762652.923722:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762652.923723:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762652/real 1789762652] req@ffff8d25d25b8000 x1876699507676800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762652.923726:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762652.923739:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507677440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762652.923752:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507676800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762652.923762:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762652.923765:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762652.923768:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507676800:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762652.923786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762652.923805:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762652.923814:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762652/real 1789762652] req@ffff8d25d25ba300 x1876699507676928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762652.923823:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507676928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762652.923833:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762652.923835:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762652.923838:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507676928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762652.923841:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762652.923852:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762652.923858:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762652/real 1789762652] req@ffff8d25d25b8a80 x1876699507677056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762652.923865:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507677056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762652.923870:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762652.923872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762652.923874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762652.923877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762652.923888:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762652.923894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762652/real 1789762652] req@ffff8d25d25b9180 x1876699507677184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762652.923902:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507677184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762652.923907:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762652.923909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762652.923911:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762652.923915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507677312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762652.923924:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762652.923930:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762652/real 1789762652] req@ffff8d25d25bbb80 x1876699507677312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762652.923937:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507677312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762652.923942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762652.923943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762652.923945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507677312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762652.923948:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507677440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762652.923958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762652.923964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762652/real 1789762652] req@ffff8d25d25bad80 x1876699507677440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762652.923971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507677440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762677 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762652.923976:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762652.923978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762652.923980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507677440:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762652.987480:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762654.011492:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762655.035493:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762656.059547:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762657.083509:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762658.043529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762658.043544:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762658.043550:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.043562:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2635 00000100:00080000:2.0:1789762658.043573:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.043580:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762658.043625:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507677568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762658.043659:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762658.043664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762658.043666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.043672:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2635 00000100:00080000:2.0:1789762658.043676:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.043680:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762658.043693:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762658.043694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507677568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762658.043716:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507677696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762658.043730:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762658.043733:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762658.043735:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.043739:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2635 00000100:00080000:2.0:1789762658.043756:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.043761:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762658.043766:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762658.043774:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762658.043776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507677824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762658.043784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762658.043788:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762658.043790:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.043793:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2635 00000100:00080000:2.0:1789762658.043796:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.043800:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762658.043804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762658.043814:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507677952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762658.043824:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762658.043824:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffff8d25c8023b80 x1876699507677568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762658.043828:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762658.043831:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.043835:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2635 00000100:00080000:2.0:1789762658.043838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.043841:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762658.043845:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507677568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762658.043846:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762658.043855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507678080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762658.043857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762658.043861:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762658.043864:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762658.043866:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507677568:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762658.043868:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762658.043870:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762658.043873:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2635 00000100:00080000:2.0:1789762658.043876:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762658.043879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762658.043884:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762658.043892:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762658.043893:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507678208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762658.043915:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762658.043929:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffff8d25c8022300 x1876699507677696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762658.043942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507677696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762658.043950:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762658.043953:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762658.043955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762658.043959:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762658.043975:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762658.043983:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffff8d25c8020380 x1876699507677824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762658.043995:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507677824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762658.044001:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762658.044002:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762658.044004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762658.044008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762658.044019:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762658.044026:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffff8d25c8022d80 x1876699507677952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762658.044033:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507677952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762658.044039:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762658.044040:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762658.044042:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507677952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762658.044045:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762658.044055:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762658.044069:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffff8d25c8023100 x1876699507678080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762658.044080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507678080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762658.044089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762658.044092:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762658.044094:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762658.044105:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762658.044118:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762658.044125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762658/real 1789762658] req@ffff8d25c8023480 x1876699507678208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762658.044132:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507678208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762683 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762658.044137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762658.044139:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762658.044141:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678208:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762658.107516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762659.131587:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762660.155598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762661.179530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762662.203483:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762663.163509:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762663.163520:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762663.163526:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762663.163539:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2640 00000100:00080000:3.0:1789762663.163552:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762663.163561:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762663.163606:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507678336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762663.163654:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762663.163658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762663.163660:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762663.163665:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2640 00000100:00080000:3.0:1789762663.163669:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762663.163680:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762663.163692:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762663.163708:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507678464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762663.163720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762663.163725:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762663.163727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762663.163733:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2640 00000100:00080000:3.0:1789762663.163739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762663.163745:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507678336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762663.163747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762663.163757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762663.163771:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507678592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762663.163848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762663.163851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762663.163853:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762663.163857:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2640 00000100:00080000:3.0:1789762663.163861:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762663.163867:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762663.163869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507678464:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789762663.163873:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762663.163888:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507678720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762663.163914:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507678592:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762663.164025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762663.164029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762663.164031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762663.164037:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2640 00000100:00080000:3.0:1789762663.164043:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762663.164048:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762663.164055:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762663.164062:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507678720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762663.164073:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507678848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762663.164087:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762663.164092:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762663.164094:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762663.164099:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2640 00000100:00080000:3.0:1789762663.164104:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762663.164112:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678848:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762663.164112:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762663.164123:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762663.164139:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507678976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762663.164193:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678976:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762663.227543:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762664.251487:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762665.275519:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789762666.235669:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789762666.235717:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789762666.235735:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762666.235781:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762666.235792:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762666.235806:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762666.235817:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762666.235826:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789762666.235841:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762666] req@ffff8d25d1b08000 x1876699507678848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762666.235873:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507678848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762666.235886:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762666.235892:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762666.235897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762666.235906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762666] req@ffff8d25d1b09500 x1876699507678720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762666.235920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507678720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762666.235928:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762666.235932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762666.235936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507678720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762666.235943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762666] req@ffff8d25d1b08380 x1876699507678592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762666.235955:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507678592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762666.235964:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762666.235968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762666.235972:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507678592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762666.235980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762666] req@ffff8d25d1b0b100 x1876699507678336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762666.235992:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507678336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762666.236000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762666.236003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762666.236006:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507678336:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762666.236013:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762666] req@ffff8d25d1b0a300 x1876699507678464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762666.236024:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507678464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762666.236031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762666.236035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762666.236038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507678464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762666.236081:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762663/real 1789762666] req@ffff8d25d1b0b480 x1876699507678976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762666.236095:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507678976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762688 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762666.236104:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762666.236107:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762666.236110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507678976:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762666.299515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762667.323503:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762668.283508:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.283516:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762668.283519:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.283526:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2645 00000100:00080000:0.0:1789762668.283533:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.283539:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762668.283583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507679104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762668.283607:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.283610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762668.283611:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.283614:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2645 00000100:00080000:0.0:1789762668.283617:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.283620:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762668.283628:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762668.283642:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507679232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762668.283642:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507679104:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762668.283661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.283664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762668.283666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.283669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2645 00000100:00080000:0.0:1789762668.283671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.283674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762668.283677:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762668.283689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507679360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762668.283695:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.283697:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762668.283699:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.283701:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2645 00000100:00080000:0.0:1789762668.283703:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.283705:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762668.283708:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762668.283715:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762668.283718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507679488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762668.283727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.283730:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762668.283731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.283734:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2645 00000100:00080000:0.0:1789762668.283743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.283746:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762668.283749:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762668.283757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507679616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762668.283764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762668.283767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762668.283768:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffff8d25d25ba680 x1876699507679104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762668.283769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762668.283775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2645 00000100:00080000:0.0:1789762668.283780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762668.283787:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762668.283792:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762668.283804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507679744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762668.283808:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507679104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762668.283833:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762668.283838:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762668.283843:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507679104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762668.283871:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507679232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762668.283898:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762668.283911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffff8d25d25bb800 x1876699507679232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762668.283924:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507679232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762668.283934:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762668.283938:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762668.283942:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507679232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762668.283947:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507679360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762668.283968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762668.283979:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffff8d25d25b9f80 x1876699507679360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762668.283990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507679360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762668.283997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762668.284000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762668.284003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507679360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762668.284009:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507679488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762668.284026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762668.284037:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffff8d25d25b8380 x1876699507679488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762668.284049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507679488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762668.284057:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762668.284060:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762668.284063:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507679488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762668.284069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507679616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762668.284088:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762668.284100:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffff8d25d25b8700 x1876699507679616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762668.284120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507679616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762668.284128:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762668.284131:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762668.284135:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507679616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762668.284141:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507679744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762668.284163:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762668.284175:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762668/real 1789762668] req@ffff8d25d25bb100 x1876699507679744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762668.284186:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507679744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762693 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762668.284194:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762668.284198:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762668.284201:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507679744:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762668.347533:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762669.371516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762670.395546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762671.419668:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762672.443525:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762673.403573:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762673.403585:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762673.403590:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762673.403602:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2651 00000100:00080000:1.0:1789762673.403616:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762673.403624:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762673.403675:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507679872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762673.403708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762673.403711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762673.403713:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762673.403717:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2651 00000100:00080000:1.0:1789762673.403720:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762673.403723:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762673.403734:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762673.403749:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507680000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762673.403762:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762673.403769:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762673.403770:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507679872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762673.403772:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762673.403781:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2651 00000100:00080000:1.0:1789762673.403786:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762673.403791:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762673.403797:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762673.403808:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507680128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762673.403816:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762673.403819:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762673.403821:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762673.403824:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2651 00000100:00080000:1.0:1789762673.403828:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762673.403830:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762673.403834:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762673.403842:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507680256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762673.403849:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762673.403852:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762673.403853:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762673.403858:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762673.403859:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2651 00000100:00080000:1.0:1789762673.403864:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762673.403867:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762673.403871:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762673.403879:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507680384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762673.403886:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762673.403888:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762673.403890:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762673.403893:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2651 00000100:00080000:1.0:1789762673.403895:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762673.403897:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762673.403901:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762673.403908:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507680512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762673.403918:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762673/real 1789762673] req@ffff8d25c8169500 x1876699507679872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762673.403936:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507679872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762673.403947:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762673.403951:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762673.403955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507679872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762673.403980:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762673.403996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762673.404005:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762673/real 1789762673] req@ffff8d25c8168000 x1876699507680000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762673.404014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507680000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762673.404020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762673.404022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762673.404025:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762673.404028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762673.404039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762673.404045:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762673/real 1789762673] req@ffff8d25c8168a80 x1876699507680128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762673.404053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507680128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762673.404059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762673.404060:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762673.404062:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762673.404066:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762673.404078:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762673.404084:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762673/real 1789762673] req@ffff8d25c816a300 x1876699507680256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762673.404092:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507680256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762673.404098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762673.404100:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762673.404101:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762673.404105:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507680384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762673.404115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762673.404121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762673/real 1789762673] req@ffff8d25c816ad80 x1876699507680384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762673.404129:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507680384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762673.404134:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762673.404136:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762673.404138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507680384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762673.404141:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507680512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762673.404151:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762673.404157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762673/real 1789762673] req@ffff8d25c8169c00 x1876699507680512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762673.404174:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507680512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762698 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762673.404180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762673.404182:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762673.404184:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507680512:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762673.467572:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762674.491521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762675.515483:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762676.539527:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762677.563589:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762678.523511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762678.523522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762678.523527:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762678.523540:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2656 00000100:00080000:3.0:1789762678.523552:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762678.523561:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762678.523611:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507680640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762678.523642:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762678.523645:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762678.523646:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762678.523650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2656 00000100:00080000:3.0:1789762678.523653:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762678.523657:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762678.523666:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762678.523677:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507680768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762678.523684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762678.523687:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762678.523688:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762678.523691:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2656 00000100:00080000:3.0:1789762678.523693:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762678.523695:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762678.523698:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762678.523702:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507680640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762678.523709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507680896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762678.523717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762678.523720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762678.523721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762678.523726:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2656 00000100:00080000:3.0:1789762678.523729:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762678.523733:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762678.523739:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762678.523753:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507681024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762678.523765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762678.523769:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762678.523771:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762678.523779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2656 00000800:00000100:0.0:1789762678.523780:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762678.523786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762678.523792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762678.523797:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762678.523806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507681152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762678.523813:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762678.523816:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762678.523817:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762678.523819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2656 00000100:00080000:3.0:1789762678.523822:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762678.523824:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762678.523828:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762678.523843:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762678/real 1789762678] req@ffff8d25d1b09c00 x1876699507680640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762678.523843:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507681280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762678.523884:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507680640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762678.523893:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762678.523898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762678.523902:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507680640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762678.523909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762678.523926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762678.523935:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762678/real 1789762678] req@ffff8d25d1b0ad80 x1876699507680768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762678.523944:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507680768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762678.523949:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762678.523951:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762678.523953:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762678.523981:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762678.523993:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762678.524000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762678/real 1789762678] req@ffff8d25d1b09180 x1876699507680896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762678.524008:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507680896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762678.524013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762678.524015:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762678.524017:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507680896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762678.524021:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762678.524032:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762678.524038:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762678/real 1789762678] req@ffff8d25d1b0aa00 x1876699507681024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762678.524046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507681024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762678.524052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762678.524053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762678.524055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762678.524058:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507681152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762678.524068:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762678.524075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762678/real 1789762678] req@ffff8d25d1b08a80 x1876699507681152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762678.524083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507681152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762678.524088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762678.524089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762678.524091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507681152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762678.524094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507681280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762678.524104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762678.524109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762678/real 1789762678] req@ffff8d25d1b0bb80 x1876699507681280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762678.524116:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507681280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762703 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762678.524126:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762678.524128:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762678.524130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507681280:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762678.587507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762679.611518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762680.635570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762681.659563:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762682.683582:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762683.643471:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762683.643479:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762683.643483:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762683.643492:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2661 00000100:00080000:3.0:1789762683.643501:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762683.643507:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762683.643539:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507681408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762683.643563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762683.643566:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762683.643567:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762683.643571:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2661 00000100:00080000:3.0:1789762683.643573:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762683.643575:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762683.643582:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762683.643589:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507681536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762683.643596:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762683.643599:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762683.643600:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762683.643603:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2661 00000100:00080000:3.0:1789762683.643605:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762683.643607:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762683.643610:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762683.643616:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507681664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762683.643622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762683.643625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762683.643626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762683.643632:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2661 00000100:00100000:0.0:1789762683.643633:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507681408:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762683.643635:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762683.643642:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762683.643647:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762683.643655:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507681792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762683.643662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762683.643665:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762683.643666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762683.643669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2661 00000100:00080000:3.0:1789762683.643671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762683.643673:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762683.643677:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762683.643683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507681920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762683.643689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762683.643692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762683.643693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762683.643697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762683.643701:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2661 00000100:00080000:3.0:1789762683.643705:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762683.643708:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762683.643712:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762683.643721:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507682048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762683.643741:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762683/real 1789762683] req@ffff8d25d1b09880 x1876699507681408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762683.643756:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507681408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762683.643763:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762683.643766:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762683.643769:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507681408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762683.643774:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762683.643788:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762683.643797:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762683/real 1789762683] req@ffff8d25d1b08700 x1876699507681536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762683.643805:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507681536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762683.643810:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762683.643812:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762683.643814:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762683.643836:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762683.643847:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762683.643854:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762683/real 1789762683] req@ffff8d25d1b0a680 x1876699507681664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762683.643861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507681664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762683.643866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762683.643868:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762683.643870:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762683.643874:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762683.643884:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762683.643890:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762683/real 1789762683] req@ffff8d25d1b0b800 x1876699507681792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762683.643897:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507681792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762683.643902:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762683.643904:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762683.643906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507681792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762683.643909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507681920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762683.643918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762683.643925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762683/real 1789762683] req@ffff8d25d1b08e00 x1876699507681920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762683.643933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507681920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762683.643938:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762683.643940:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762683.643942:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507681920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762683.643945:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507682048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762683.643954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762683.643960:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762683/real 1789762683] req@ffff8d25d1b0bb80 x1876699507682048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762683.643967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507682048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762708 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762683.643972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762683.643974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762683.643976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507682048:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762683.707500:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762684.731599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762685.755593:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762686.779604:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762687.803521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762688.763523:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762688.763537:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762688.763543:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762688.763557:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2666 00000100:00080000:0.0:1789762688.763569:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762688.763580:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762688.763624:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507682176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762688.763652:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762688.763656:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762688.763657:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762688.763660:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2666 00000100:00080000:0.0:1789762688.763664:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762688.763667:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762688.763678:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762688.763687:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507682304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762688.763694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507682176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762688.763697:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762688.763703:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762688.763704:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762688.763718:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2666 00000100:00080000:0.0:1789762688.763722:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762688.763727:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762688.763735:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762688.763749:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507682432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762688.763760:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762688.763764:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762688.763767:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762688.763771:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762688.763775:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2666 00000100:00080000:0.0:1789762688.763782:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762688.763786:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762688.763793:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762688.763810:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507682560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762688.763820:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762688.763822:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762688/real 1789762688] req@ffff8d25d25b9c00 x1876699507682176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762688.763824:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762688.763828:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762688.763837:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2666 00000100:00080000:0.0:1789762688.763841:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762688.763845:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762688.763849:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507682176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762688.763851:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762688.763864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762688.763867:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507682688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762688.763870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762688.763876:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507682176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762688.763885:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762688.763888:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762688.763889:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762688.763894:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2666 00000100:00080000:0.0:1789762688.763897:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762688.763899:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762688.763902:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762688.763907:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507682304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762688.763913:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507682816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762688.763930:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762688.763943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762688/real 1789762688] req@ffff8d25d25b8e00 x1876699507682304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762688.763952:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507682304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762688.763957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762688.763960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762688.763962:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507682304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762688.763966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507682432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762688.763980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762688.763986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762688/real 1789762688] req@ffff8d25d25bb480 x1876699507682432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762688.763994:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507682432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762688.764000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762688.764002:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762688.764004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507682432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762688.764008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507682560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762688.764019:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762688.764025:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762688/real 1789762688] req@ffff8d25d25bb100 x1876699507682560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762688.764033:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507682560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762688.764038:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762688.764040:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762688.764042:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507682560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762688.764045:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507682688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762688.764055:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762688.764075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762688/real 1789762688] req@ffff8d25d25b8700 x1876699507682688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762688.764096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507682688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762688.764105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762688.764109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762688.764113:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507682688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762688.764130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507682816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762688.764146:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762688.764156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762688/real 1789762688] req@ffff8d25d25b8380 x1876699507682816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762688.764169:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507682816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762713 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762688.764178:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762688.764182:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762688.764185:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507682816:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762688.827545:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762689.851545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762690.875585:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762691.899531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762692.923598:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762693.883509:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762693.883519:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762693.883524:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762693.883536:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2671 00000100:00080000:2.0:1789762693.883547:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762693.883555:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762693.883593:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507682944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762693.883622:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762693.883626:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762693.883628:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762693.883633:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2671 00000100:00080000:2.0:1789762693.883638:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762693.883643:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762693.883653:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762693.883666:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507682944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762693.883669:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507683072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762693.883691:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762693.883695:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762693.883697:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762693.883701:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2671 00000100:00080000:2.0:1789762693.883706:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762693.883710:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762693.883716:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762693.883727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507683200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762693.883739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762693.883743:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762693.883745:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762693.883749:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2671 00000100:00080000:2.0:1789762693.883761:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762693.883766:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762693.883772:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762693.883777:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762693.883789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507683328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762693.883803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762693.883807:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762693.883809:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762693.883814:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2671 00000100:00080000:2.0:1789762693.883818:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762693.883823:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762693.883828:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762693.883840:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762693/real 1789762693] req@ffff8d25c8021c00 x1876699507682944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762693.883847:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507683456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762693.883875:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762693.883882:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762693.883882:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507682944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762693.883897:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762693.883902:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762693.883904:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2671 00000100:00080000:2.0:1789762693.883915:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762693.883919:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762693.883925:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762693.883927:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507682944:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789762693.883936:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762693.883961:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507683584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762693.883974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762693.884004:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762693.884015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762693/real 1789762693] req@ffff8d25c8021500 x1876699507683072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762693.884025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507683072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762693.884031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762693.884034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762693.884036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762693.884041:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762693.884053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762693.884060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762693/real 1789762693] req@ffff8d25c8020e00 x1876699507683200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762693.884067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507683200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762693.884073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762693.884074:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762693.884076:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762693.884081:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762693.884092:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762693.884099:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762693/real 1789762693] req@ffff8d25c8020700 x1876699507683328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762693.884107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507683328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762693.884112:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762693.884114:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762693.884116:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762693.884119:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507683456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762693.884130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762693.884136:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762693/real 1789762693] req@ffff8d25c8021880 x1876699507683456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762693.884144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507683456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762693.884149:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762693.884151:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762693.884152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507683456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762693.884164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507683584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762693.884175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762693.884181:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762693/real 1789762693] req@ffff8d25c8023480 x1876699507683584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762693.884190:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507683584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762718 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762693.884195:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762693.884198:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762693.884199:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507683584:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762693.947528:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762694.971523:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762695.995531:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762697.019617:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762698.043562:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762699.003495:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762699.003507:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762699.003513:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762699.003526:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2676 00000100:00080000:0.0:1789762699.003539:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762699.003551:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762699.003604:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507683712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762699.003642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762699.003647:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762699.003651:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762699.003661:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2676 00000100:00080000:0.0:1789762699.003667:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762699.003673:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762699.003677:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507683712:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762699.003697:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762699.003716:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507683840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762699.003727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762699.003731:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762699.003735:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762699.003739:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2676 00000100:00080000:0.0:1789762699.003753:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762699.003758:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789762699.003763:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762699.003767:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762699.003784:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507683968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762699.003803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762699.003807:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762699.003810:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762699.003815:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2676 00000100:00080000:0.0:1789762699.003820:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762699.003825:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762699.003831:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762699.003836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffff8d25d25b9f80 x1876699507683712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762699.003847:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507684096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762699.003856:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507683712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762699.003871:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762699.003877:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762699.003878:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762699.003884:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789762699.003885:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507683712:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762699.003887:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762699.003894:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2676 00000100:00080000:0.0:1789762699.003898:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762699.003901:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762699.003906:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762699.003916:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507684224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762699.003923:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762699.003926:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762699.003927:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762699.003929:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2676 00000100:00080000:0.0:1789762699.003931:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762699.003933:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762699.003936:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762699.003942:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507684352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762699.003950:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762699.003974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762699.003986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffff8d25d25bb800 x1876699507683840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762699.003996:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507683840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762699.004003:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762699.004006:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762699.004009:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762699.004013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762699.004042:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762699.004055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffff8d25d25ba680 x1876699507683968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762699.004067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507683968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762699.004074:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762699.004077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762699.004082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507683968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762699.004087:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762699.004104:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762699.004111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffff8d25d25bad80 x1876699507684096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762699.004119:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507684096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762699.004125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762699.004127:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762699.004131:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762699.004136:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507684224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762699.004152:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762699.004171:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffff8d25d25bbb80 x1876699507684224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762699.004190:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507684224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762699.004200:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762699.004203:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762699.004209:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507684224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762699.004246:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507684352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762699.004280:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762699.004306:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762699/real 1789762699] req@ffff8d25d25b9180 x1876699507684352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762699.004317:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507684352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762724 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762699.004324:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762699.004326:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762699.004328:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507684352:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762699.067572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762700.091515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762701.115512:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762702.139568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762703.163514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762704.123464:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762704.123472:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762704.123475:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762704.123484:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2681 00000100:00080000:1.0:1789762704.123492:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762704.123498:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762704.123528:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507684480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762704.123550:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762704.123552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762704.123554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762704.123557:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2681 00000100:00080000:1.0:1789762704.123560:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762704.123563:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762704.123570:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762704.123578:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507684608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762704.123586:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762704.123588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762704.123589:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762704.123593:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2681 00000100:00080000:1.0:1789762704.123595:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762704.123598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762704.123601:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762704.123609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507684736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762704.123617:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762704.123619:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762704.123620:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762704.123622:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2681 00000100:00080000:1.0:1789762704.123625:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762704.123629:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762704.123637:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762704.123638:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507684480:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762704.123646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507684864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762704.123660:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762704.123662:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762704.123664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762704.123666:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2681 00000100:00080000:1.0:1789762704.123668:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762704.123670:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762704.123674:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762704.123683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507684992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762704.123689:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762704.123691:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762704.123692:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762704.123695:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2681 00000100:00080000:1.0:1789762704.123697:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762704.123699:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762704.123701:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762704.123708:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507685120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762704.123750:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762704.123823:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffff8d25c816b480 x1876699507684480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762704.123845:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507684480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762704.123859:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762704.123865:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762704.123870:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507684480:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762704.123887:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762704.123914:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762704.123929:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffff8d25c8169880 x1876699507684608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762704.123943:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507684608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762704.123952:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762704.123956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762704.123959:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762704.123991:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762704.124015:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762704.124028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffff8d25c8169c00 x1876699507684736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762704.124048:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507684736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762704.124057:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762704.124061:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762704.124065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762704.124072:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762704.124094:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762704.124107:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffff8d25c816ad80 x1876699507684864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762704.124120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507684864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762704.124130:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762704.124134:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762704.124137:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507684864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762704.124144:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507684992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762704.124165:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762704.124178:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffff8d25c816a300 x1876699507684992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762704.124190:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507684992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762704.124207:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762704.124211:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762704.124214:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507684992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762704.124221:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507685120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762704.124243:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762704.124255:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762704/real 1789762704] req@ffff8d25c8168a80 x1876699507685120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762704.124268:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507685120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762729 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762704.124277:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762704.124280:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762704.124284:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507685120:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762704.187525:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762705.211501:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762706.235580:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762707.259586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762708.283536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762709.243455:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762709.243463:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762709.243472:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762709.243481:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2686 00000100:00080000:1.0:1789762709.243494:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762709.243505:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762709.243546:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507685248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762709.243576:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762709.243580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762709.243582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762709.243587:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2686 00000100:00080000:1.0:1789762709.243592:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762709.243596:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762709.243606:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762709.243622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507685376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762709.243635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762709.243639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762709.243643:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762709.243650:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507685248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762709.243651:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2686 00000100:00080000:1.0:1789762709.243660:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762709.243668:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762709.243674:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762709.243692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507685504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762709.243703:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762709.243707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762709.243710:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762709.243715:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2686 00000100:00080000:1.0:1789762709.243720:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762709.243725:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762709.243731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789762709.243748:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762709.243755:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507685632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762709.243770:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762709.243774:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762709.243777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762709.243783:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2686 00000100:00080000:1.0:1789762709.243789:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762709.243794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762709.243800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762709.243802:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffff8d25c8168000 x1876699507685248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762709.243820:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507685760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762709.243823:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507685248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762709.243868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762709.243872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762709.243875:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762709.243879:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762709.243881:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762709.243886:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507685248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762709.243893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2686 00000100:00080000:1.0:1789762709.243900:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762709.243905:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762709.243912:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762709.243924:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507685376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762709.243929:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507685888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762709.243960:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762709.243975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffff8d25c8169500 x1876699507685376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762709.243985:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507685376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762709.243992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762709.243996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762709.244000:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507685376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762709.244007:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507685504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762709.244030:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762709.244043:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffff8d25c8168700 x1876699507685504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762709.244054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507685504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762709.244060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762709.244063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762709.244065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507685504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762709.244069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507685632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762709.244085:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762709.244094:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffff8d25c8168380 x1876699507685632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762709.244107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507685632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762709.244115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762709.244119:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762709.244122:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507685632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762709.244129:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507685760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762709.244150:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762709.244163:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffff8d25c8168e00 x1876699507685760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762709.244177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507685760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762709.244186:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762709.244189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762709.244193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507685760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762709.244213:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507685888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762709.244230:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762709.244238:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762709/real 1789762709] req@ffff8d25c8169f80 x1876699507685888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762709.244247:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507685888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762734 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762709.244253:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762709.244255:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762709.244257:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507685888:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762709.307554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762710.331541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762711.355549:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762712.379516:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762713.403511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762714.363486:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762714.363493:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762714.363497:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762714.363504:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2692 00000100:00080000:0.0:1789762714.363512:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762714.363517:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762714.363548:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507686016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762714.363571:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762714.363574:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762714.363576:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762714.363579:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2692 00000100:00080000:0.0:1789762714.363581:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762714.363585:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762714.363594:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762714.363604:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507686144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762714.363613:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762714.363616:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762714.363619:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762714.363624:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507686016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762714.363625:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2692 00000100:00080000:0.0:1789762714.363633:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762714.363643:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762714.363648:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762714.363664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507686272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762714.363675:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762714.363679:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762714.363680:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762714.363686:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2692 00000100:00080000:0.0:1789762714.363691:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762714.363695:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762714.363701:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762714.363714:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507686400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762714.363714:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762714.363737:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762714.363740:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762714.363741:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762714.363745:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2692 00000100:00080000:0.0:1789762714.363754:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762714.363757:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762714.363761:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762714.363769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507686528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762714.363781:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762714.363785:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789762714.363786:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffff8d25d25b8a80 x1876699507686016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762714.363788:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762714.363803:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2692 00000100:00080000:0.0:1789762714.363815:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762714.363820:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762714.363826:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762714.363832:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507686016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762714.363838:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507686656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762714.363864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762714.363872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762714.363876:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507686016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762714.363903:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762714.363923:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762714.363933:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffff8d25d25ba300 x1876699507686144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762714.363942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507686144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762714.363948:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762714.363951:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762714.363954:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762714.363960:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762714.363980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762714.363993:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffff8d25d25b8000 x1876699507686272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762714.364007:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507686272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762714.364014:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762714.364016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762714.364019:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762714.364024:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762714.364041:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762714.364049:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffff8d25d25b9180 x1876699507686400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762714.364057:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507686400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762714.364063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762714.364065:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762714.364067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762714.364071:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507686528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762714.364083:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762714.364090:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffff8d25d25bbb80 x1876699507686528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762714.364098:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507686528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762714.364103:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762714.364105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762714.364106:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507686528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762714.364110:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507686656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762714.364120:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762714.364127:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762714/real 1789762714] req@ffff8d25d25bad80 x1876699507686656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762714.364134:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507686656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762739 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762714.364140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762714.364141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762714.364143:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507686656:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762714.427506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762715.451512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762716.475509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762717.499576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762718.523564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762719.483566:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762719.483579:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762719.483584:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762719.483598:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2697 00000100:00080000:0.0:1789762719.483609:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762719.483619:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762719.483666:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507686784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762719.483705:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762719.483711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762719.483713:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762719.483719:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2697 00000100:00080000:0.0:1789762719.483724:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762719.483729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762719.483741:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762719.483759:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507686912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762719.483774:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762719.483783:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789762719.483785:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507686784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762719.483788:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762719.483807:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2697 00000100:00080000:0.0:1789762719.483811:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762719.483816:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762719.483822:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762719.483842:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507687040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762719.483853:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762719.483858:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762719.483861:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762719.483865:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2697 00000100:00080000:0.0:1789762719.483869:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762719.483873:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762719.483881:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762719.483895:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507687168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762719.483898:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762719.483919:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762719.483924:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762719.483926:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762719.483931:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2697 00000100:00080000:0.0:1789762719.483937:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762719.483943:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762719.483953:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762719.483974:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507687296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762719.483988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762719/real 1789762719] req@ffff8d25d25ba680 x1876699507686784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762719.484002:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762719.484016:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762719.484020:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762719.484029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507686784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762719.484033:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2697 00000100:00080000:0.0:1789762719.484054:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762719.484059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762719.484068:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789762719.484068:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762719.484087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507686784:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762719.484090:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762719.484114:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507687424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762719.484127:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762719.484177:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762719.484208:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762719/real 1789762719] req@ffff8d25d25bb800 x1876699507686912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762719.484220:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507686912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762719.484229:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762719.484232:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762719.484234:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507686912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762719.484240:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762719.484257:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762719.484265:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762719/real 1789762719] req@ffff8d25d25b9f80 x1876699507687040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762719.484274:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507687040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762719.484284:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762719.484287:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762719.484289:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762719.484293:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762719.484305:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762719.484311:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762719/real 1789762719] req@ffff8d25d25b8380 x1876699507687168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762719.484319:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507687168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762719.484325:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762719.484327:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762719.484329:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762719.484333:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507687296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762719.484345:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762719.484352:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762719/real 1789762719] req@ffff8d25d25b8700 x1876699507687296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762719.484360:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507687296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762719.484365:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762719.484367:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762719.484370:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507687296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762719.484422:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507687424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762719.484437:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762719.484445:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762719/real 1789762719] req@ffff8d25d25bb100 x1876699507687424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762719.484453:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507687424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762744 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762719.484459:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762719.484462:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762719.484464:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507687424:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762719.547639:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762720.571520:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762721.595598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762722.619504:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762723.643603:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762724.603477:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762724.603488:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762724.603492:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762724.603503:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2702 00000100:00080000:2.0:1789762724.603513:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762724.603522:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762724.603560:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507687552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762724.603588:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762724.603591:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762724.603593:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762724.603596:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2702 00000100:00080000:2.0:1789762724.603601:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762724.603605:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762724.603615:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762724.603621:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507687552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762724.603629:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507687680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762724.603642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762724.603646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762724.603648:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762724.603652:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2702 00000100:00080000:2.0:1789762724.603657:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762724.603662:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762724.603668:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762724.603682:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507687808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762724.603693:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762724.603696:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762724.603698:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762724.603701:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2702 00000100:00080000:2.0:1789762724.603705:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762724.603706:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762724.603711:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762724.603717:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762724.603727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507687936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762724.603740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762724.603743:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762724.603744:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762724.603746:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2702 00000100:00080000:2.0:1789762724.603750:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762724.603755:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762724.603760:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762724/real 1789762724] req@ffff8d25c8023100 x1876699507687552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762724.603763:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762724.603784:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507688064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762724.603785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507687552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762724.603819:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762724.603820:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762724.603826:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762724.603829:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762724.603830:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762724.603836:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507687552:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762724.603838:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2702 00000100:00080000:2.0:1789762724.603845:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762724.603850:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762724.603855:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762724.603865:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507688192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762724.603868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762724.603895:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762724.603908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762724/real 1789762724] req@ffff8d25c8022d80 x1876699507687680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762724.603916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507687680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762724.603922:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762724.603924:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762724.603927:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762724.603931:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762724.603943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762724.603949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762724/real 1789762724] req@ffff8d25c8020380 x1876699507687808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762724.603957:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507687808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762724.603962:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762724.603964:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762724.603966:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762724.603970:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762724.603981:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762724.603987:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762724/real 1789762724] req@ffff8d25c8022300 x1876699507687936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762724.603995:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507687936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762724.604001:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762724.604003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762724.604004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507687936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762724.604008:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762724.604018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762724.604025:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762724/real 1789762724] req@ffff8d25c8023b80 x1876699507688064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762724.604032:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507688064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762724.604037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762724.604039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762724.604041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762724.604051:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762724.604062:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762724.604068:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762724/real 1789762724] req@ffff8d25c8020000 x1876699507688192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762724.604074:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507688192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762749 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762724.604084:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762724.604086:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762724.604088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688192:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762724.667549:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762725.691586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762726.715636:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762727.739585:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762728.763604:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762729.723463:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762729.723474:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762729.723480:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762729.723490:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2707 00000100:00080000:2.0:1789762729.723500:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762729.723507:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762729.723549:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507688320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762729.723575:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762729.723578:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762729.723579:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762729.723583:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2707 00000100:00080000:2.0:1789762729.723586:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762729.723590:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762729.723600:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762729.723609:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507688320:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762729.723615:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507688448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762729.723624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762729.723626:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762729.723628:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762729.723630:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2707 00000100:00080000:2.0:1789762729.723633:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762729.723636:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762729.723639:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762729.723649:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507688576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762729.723661:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762729.723664:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762729.723665:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762729.723667:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2707 00000100:00080000:2.0:1789762729.723670:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762729.723672:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762729.723676:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762729.723684:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507688704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762729.723691:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762729.723693:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762729.723695:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762729.723702:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2707 00000100:00080000:2.0:1789762729.723705:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762729.723708:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762729.723717:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762729.723725:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507688832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762729.723730:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507688448:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789762729.723739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762729.723742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762729.723744:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762729.723748:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2707 00000100:00080000:2.0:1789762729.723753:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762729.723758:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762729.723765:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762729.723767:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507688576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762729.723785:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507688960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762729.723790:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507688704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762729.723816:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762729.723832:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688960:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762729.787587:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762730.811535:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762731.835665:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789762732.795761:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789762732.795821:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762732.795844:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762732.795897:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762732.795908:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762732.795918:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762732.795932:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762732.795941:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789762732.795990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762729/real 1789762732] req@ffff8d25c8022a00 x1876699507688832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762732.796052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507688832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762732.796073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762732.796081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762732.796110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762732.796128:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762729/real 1789762732] req@ffff8d25c8020000 x1876699507688960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762732.796150:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507688960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762732.796163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762732.796168:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762732.796174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507688960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762732.796184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762729/real 1789762732] req@ffff8d25c8020a80 x1876699507688448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762732.796203:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507688448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762732.796215:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762732.796220:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762732.796225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507688448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762732.796236:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762729/real 1789762732] req@ffff8d25c8021180 x1876699507688576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762732.796252:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507688576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762732.796265:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762732.796271:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762732.796277:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507688576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762732.796289:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762729/real 1789762732] req@ffff8d25c8022680 x1876699507688704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762732.796311:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507688704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762732.796325:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762732.796331:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762732.796337:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507688704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762732.796350:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762729/real 1789762732] req@ffff8d25c8023800 x1876699507688320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762732.796372:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507688320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762754 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762732.796428:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762732.796436:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762732.796443:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507688320:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789762732.859639:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762733.883614:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762734.843547:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762734.843567:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762734.843575:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762734.843595:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2712 00000100:00080000:2.0:1789762734.843613:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762734.843626:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762734.843686:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507689088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762734.843738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762734.843744:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762734.843747:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762734.843754:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2712 00000100:00080000:2.0:1789762734.843761:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762734.843767:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762734.843782:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762734.843802:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507689216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762734.843827:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507689088:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762734.843840:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762734.843848:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762734.843853:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762734.843862:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2712 00000100:00080000:2.0:1789762734.843874:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762734.843880:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762734.843889:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762734.843912:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507689344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762734.843924:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762734.843930:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762734.843936:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762734.843946:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2712 00000800:00000100:0.0:1789762734.843954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762734.843959:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762734.843971:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762734.843980:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762734.844001:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507689472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762734.844016:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762734.844021:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762734.844025:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762734.844033:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2712 00000100:00080000:2.0:1789762734.844042:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762734.844055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762734/real 1789762734] req@ffff8d25c8023b80 x1876699507689088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762734.844064:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762734.844093:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762734.844139:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507689600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762734.844144:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507689088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762734.844209:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762734.844218:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762734.844219:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762734.844227:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762734.844238:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762734.844245:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2712 00000100:00080000:2.0:1789762734.844257:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762734.844259:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507689088:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762734.844274:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762734.844286:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762734.844307:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507689728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762734.844320:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762734.844488:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762734.844523:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762734/real 1789762734] req@ffff8d25c8022300 x1876699507689216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762734.844552:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507689216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762734.844571:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762734.844579:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762734.844586:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762734.844601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762734.844647:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762734.844675:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762734/real 1789762734] req@ffff8d25c8020380 x1876699507689344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762734.844702:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507689344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762734.844720:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762734.844727:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762734.844734:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762734.844748:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762734.844795:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762734.844820:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762734/real 1789762734] req@ffff8d25c8022d80 x1876699507689472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762734.844847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507689472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762734.844863:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762734.844870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762734.844877:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762734.844890:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507689600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762734.844933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762734.844959:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762734/real 1789762734] req@ffff8d25c8023100 x1876699507689600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762734.844985:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507689600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762734.845000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762734.845007:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762734.845014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507689600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762734.845056:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507689728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762734.845130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762734.845158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762734/real 1789762734] req@ffff8d25c8023480 x1876699507689728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762734.845186:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507689728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762759 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762734.845203:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762734.845210:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762734.845217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507689728:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762734.907597:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762735.931604:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762736.955691:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762737.979580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762739.003567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762739.963518:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762739.963529:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762739.963533:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762739.963544:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2717 00000100:00080000:2.0:1789762739.963572:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762739.963586:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762739.963644:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507689856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762739.963707:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762739.963712:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762739.963714:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762739.963719:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2717 00000100:00080000:2.0:1789762739.963724:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762739.963729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762739.963740:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762739.963781:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507689856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762739.963788:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507689984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762739.963809:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762739.963813:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762739.963816:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762739.963821:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2717 00000100:00080000:2.0:1789762739.963828:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762739.963834:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762739.963842:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762739.963860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507690112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762739.963875:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762739.963879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762739.963906:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762739.963908:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762739.963911:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2717 00000100:00080000:2.0:1789762739.963914:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762739.963917:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762739.963922:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762739.963931:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507690240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762739.963939:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762739.963943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762739/real 1789762739] req@ffff8d25c8021880 x1876699507689856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762739.963944:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762739.963947:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762739.963955:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2717 00000100:00080000:2.0:1789762739.963960:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762739.963964:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762739.963967:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762739.963971:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507689856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762739.964001:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507690368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762739.964007:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762739.964022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762739.964064:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507689856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762739.964065:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762739.964093:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762739.964096:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762739.964101:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2717 00000100:00080000:2.0:1789762739.964105:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762739.964110:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762739.964113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689984:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789762739.964119:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762739.964137:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507690496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789762739.964139:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762739.964154:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762739/real 1789762739] req@ffff8d25c8020700 x1876699507689984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762739.964164:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507689984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762739.964171:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762739.964194:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762739.964198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507689984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762739.964207:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762739.964230:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762739.964243:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762739/real 1789762739] req@ffff8d25c8020e00 x1876699507690112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762739.964257:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507690112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762739.964267:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762739.964271:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762739.964274:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762739.964282:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762739.964301:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762739.964312:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762739/real 1789762739] req@ffff8d25c8021500 x1876699507690240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762739.964326:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507690240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762739.964335:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762739.964339:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762739.964343:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762739.964350:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507690368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762739.964368:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762739.964379:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762739/real 1789762739] req@ffff8d25c8021c00 x1876699507690368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762739.964393:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507690368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762739.964402:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762739.964453:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762739.964459:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507690368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762739.964483:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507690496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762739.964526:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762739.964542:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762739/real 1789762739] req@ffff8d25c8021880 x1876699507690496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762739.964564:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507690496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762764 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762739.964574:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762739.964578:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762739.964582:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507690496:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762740.027619:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762741.051555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762742.075587:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762743.099613:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762744.123531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762745.083476:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762745.083486:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762745.083490:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762745.083502:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2722 00000100:00080000:1.0:1789762745.083511:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762745.083519:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762745.083559:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507690624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762745.083585:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762745.083588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762745.083590:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762745.083593:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2722 00000100:00080000:1.0:1789762745.083602:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762745.083605:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762745.083615:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762745.083626:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507690752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762745.083635:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762745.083640:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789762745.083641:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507690624:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762745.083642:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762745.083650:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2722 00000100:00080000:1.0:1789762745.083657:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762745.083661:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762745.083663:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762745.083676:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507690880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762745.083684:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762745.083688:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762745.083690:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762745.083693:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2722 00000100:00080000:1.0:1789762745.083708:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762745.083711:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762745.083714:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762745.083719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762745.083729:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507691008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762745.083739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762745.083742:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762745.083744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762745.083747:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2722 00000100:00080000:1.0:1789762745.083750:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762745.083753:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762745.083757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762745.083769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507691136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762745.083770:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffff8d25c816aa00 x1876699507690624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762745.083820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762745.083823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762745.083825:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762745.083830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2722 00000100:00100000:0.0:1789762745.083831:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507690624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762745.083837:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762745.083849:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762745.083861:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762745.083864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.083872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762745.083873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507691264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762745.083883:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507690624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762745.083934:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762745.083956:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762745.083966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffff8d25c816a680 x1876699507690752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.083975:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507690752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.083981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.083983:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762745.083985:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762745.083989:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762745.084074:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762745.084083:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffff8d25c8169f80 x1876699507690880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.084096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507690880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.084102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.084104:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762745.084106:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507690880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762745.084110:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762745.084123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762745.084129:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffff8d25c8168e00 x1876699507691008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.084136:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507691008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.084141:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.084143:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762745.084145:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762745.084149:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507691136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762745.084159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762745.084165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffff8d25c8168380 x1876699507691136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.084172:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507691136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.084182:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.084184:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762745.084186:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507691136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762745.084189:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507691264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762745.084200:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762745.084206:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762745/real 1789762745] req@ffff8d25c8168700 x1876699507691264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762745.084212:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507691264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762770 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762745.084217:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762745.084219:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762745.084221:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507691264:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762745.147544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762746.171552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762747.195496:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762748.219540:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762749.243531:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762750.203445:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762750.203453:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762750.203458:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762750.203474:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2727 00000100:00080000:1.0:1789762750.203484:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762750.203493:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762750.203527:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507691392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762750.203560:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762750.203564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762750.203566:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762750.203571:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2727 00000100:00080000:1.0:1789762750.203574:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762750.203578:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762750.203591:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762750.203592:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507691392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762750.203614:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507691520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762750.203631:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762750.203635:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762750.203637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762750.203642:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2727 00000100:00080000:1.0:1789762750.203646:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762750.203649:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762750.203655:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762750.203663:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507691648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762750.203671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762750.203675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762750.203676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762750.203683:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2727 00000800:00000100:0.0:1789762750.203684:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762750.203692:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762750.203696:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762750.203700:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762750.203711:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507691776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762750.203725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762750.203729:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762750.203731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762750.203737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2727 00000100:00080000:1.0:1789762750.203741:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762750.203746:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762750] req@ffff8d25c8169500 x1876699507691392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762750.203747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762750.203757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762750.203776:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507691904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762750.203784:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507691392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762750.203788:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762750.203805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762750.203812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762750.203819:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762750.203819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2727 00000100:00080000:1.0:1789762750.203828:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762750.203830:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789762750.203837:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762750.203839:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507691392:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789762750.203843:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762750.203855:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507692032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762750.203870:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762750.203893:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762750.203903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762750] req@ffff8d25c8168000 x1876699507691520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762750.203912:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507691520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762750.203918:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762750.203921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762750.203923:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762750.203927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762750.203938:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762750.203944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762750] req@ffff8d25c8168a80 x1876699507691648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762750.203952:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507691648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762750.203970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762750.203973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762750.203976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762750.203981:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762750.203998:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762750.204006:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762750] req@ffff8d25c816a300 x1876699507691776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762750.204014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507691776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762750.204020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762750.204022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762750.204024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507691776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762750.204028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507691904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762750.204039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762750.204045:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762750] req@ffff8d25c816ad80 x1876699507691904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762750.204052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507691904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762750.204057:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762750.204059:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762750.204061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507691904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762750.204064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507692032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762750.204075:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762750.204081:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762750/real 1789762750] req@ffff8d25c8169c00 x1876699507692032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762750.204093:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507692032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762775 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762750.204098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762750.204100:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762750.204102:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507692032:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762750.267562:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762751.291514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762752.315532:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762753.339555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762754.363671:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762755.323496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.323504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762755.323508:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.323516:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2733 00000100:00080000:1.0:1789762755.323525:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.323532:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762755.323565:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507692160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762755.323589:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.323592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762755.323593:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.323597:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2733 00000100:00080000:1.0:1789762755.323599:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.323602:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762755.323610:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762755.323622:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507692160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762755.323622:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507692288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762755.323636:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.323639:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762755.323640:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.323644:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2733 00000100:00080000:1.0:1789762755.323647:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.323649:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762755.323652:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762755.323662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507692416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762755.323668:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.323671:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762755.323673:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.323675:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2733 00000100:00080000:1.0:1789762755.323678:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.323681:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789762755.323685:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789762755.323688:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762755.323698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507692544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762755.323715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.323719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762755.323721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762755.323723:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffff8d25c8169880 x1876699507692160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762755.323731:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2733 00000100:00080000:1.0:1789762755.323739:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762755.323743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507692160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762755.323747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762755.323758:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762755.323761:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.323766:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762755.323770:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507692160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762755.323777:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507692672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762755.323790:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507692288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789762755.323791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762755.323799:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762755.323802:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762755.323808:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2733 00000800:00000100:0.0:1789762755.323813:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762755.323816:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762755.323822:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762755.323826:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffff8d25c816b480 x1876699507692288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762755.323829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762755.323841:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507692288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762755.323846:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507692800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762755.323855:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.323862:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762755.323869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507692288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762755.323879:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507692416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762755.323894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762755.323902:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffff8d25c8169c00 x1876699507692416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762755.323911:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507692416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762755.323916:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.323932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762755.323936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507692416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762755.323943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507692544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762755.323975:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762755.323986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffff8d25c816ad80 x1876699507692544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762755.323994:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507692544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762755.324000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.324002:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762755.324004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507692544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762755.324018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507692672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762755.324032:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762755.324043:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffff8d25c816a300 x1876699507692672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762755.324055:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507692672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762755.324061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.324063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762755.324066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507692672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762755.324070:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507692800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762755.324083:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762755.324091:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762755/real 1789762755] req@ffff8d25c8168a80 x1876699507692800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762755.324099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507692800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762780 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762755.324103:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762755.324105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762755.324107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507692800:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762755.387568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762756.411536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762757.435503:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762758.459501:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762759.483515:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762760.443522:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762760.443534:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762760.443539:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762760.443550:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2738 00000100:00080000:0.0:1789762760.443559:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762760.443566:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762760.443601:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507692928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762760.443629:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762760.443633:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762760.443636:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762760.443642:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2738 00000100:00080000:0.0:1789762760.443646:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762760.443652:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762760.443662:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762760.443667:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507692928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762760.443678:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507693056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762760.443690:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762760.443693:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762760.443696:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762760.443700:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2738 00000100:00080000:0.0:1789762760.443704:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762760.443709:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762760.443715:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762760.443727:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507693184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762760.443738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762760.443742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762760.443744:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762760.443747:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762760.443748:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2738 00000100:00080000:0.0:1789762760.443752:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762760.443756:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762760.443761:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762760.443775:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507693312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762760.443786:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762760.443790:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762760.443791:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762760/real 1789762760] req@ffff8d25d25bb480 x1876699507692928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762760.443793:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762760.443797:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2738 00000100:00080000:0.0:1789762760.443801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762760.443806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762760.443811:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762760.443812:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507692928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762760.443822:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507693440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762760.443825:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762760.443830:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762760.443833:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762760.443835:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507692928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762760.443838:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762760.443840:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762760.443844:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2738 00000100:00080000:0.0:1789762760.443848:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762760.443852:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762760.443857:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762760.443861:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762760.443868:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507693568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762760.443908:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762760.443925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762760/real 1789762760] req@ffff8d25d25b8e00 x1876699507693056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762760.443942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507693056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762760.443951:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762760.443953:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762760.443956:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762760.443962:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762760.443981:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762760.443990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762760/real 1789762760] req@ffff8d25d25b9c00 x1876699507693184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762760.444006:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507693184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762760.444015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762760.444018:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762760.444023:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762760.444031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762760.444056:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762760.444071:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762760/real 1789762760] req@ffff8d25d25bb100 x1876699507693312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762760.444088:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507693312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762760.444098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762760.444103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762760.444107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762760.444115:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507693440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762760.444136:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762760.444148:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762760/real 1789762760] req@ffff8d25d25b8700 x1876699507693440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762760.444163:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507693440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762760.444182:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762760.444185:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762760.444187:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507693440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762760.444203:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507693568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762760.444220:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762760.444229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762760/real 1789762760] req@ffff8d25d25b8380 x1876699507693568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762760.444238:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507693568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762785 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762760.444244:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762760.444246:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762760.444249:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507693568:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762760.507551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762761.531567:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762762.555534:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762763.579572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762764.603619:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762765.563550:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762765.563560:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762765.563566:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762765.563577:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2743 00000100:00080000:0.0:1789762765.563590:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762765.563598:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762765.563645:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507693696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762765.563677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762765.563682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762765.563684:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762765.563690:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2743 00000100:00080000:0.0:1789762765.563695:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762765.563702:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762765.563714:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762765.563726:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507693696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762765.563735:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507693824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762765.563759:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762765.563762:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762765.563764:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762765.563768:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2743 00000100:00080000:0.0:1789762765.563772:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762765.563777:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762765.563782:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762765.563795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507693952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762765.563804:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762765.563817:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762765.563820:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762765.563821:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762765.563827:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2743 00000100:00080000:0.0:1789762765.563833:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762765.563839:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762765.563857:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762765.563872:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762765/real 1789762765] req@ffff8d25d25b9f80 x1876699507693696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762765.563875:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507694080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762765.563922:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762765.563923:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507693696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762765.563945:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762765.563953:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762765.563954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762765.563973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762765.563977:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2743 00000100:00080000:0.0:1789762765.563987:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762765.563988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507693696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762765.563998:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762765.564004:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762765.564022:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507694208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762765.564022:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693824:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762765.564047:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762765.564050:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762765.564053:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762765.564058:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2743 00000100:00080000:0.0:1789762765.564061:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762765.564063:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762765.564067:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762765.564070:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762765.564080:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507694336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762765.564098:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762765/real 1789762765] req@ffff8d25d25bb800 x1876699507693824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762765.564117:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507693824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762765.564125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762765.564128:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762765.564130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762765.564136:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762765.564156:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762765.564164:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762765/real 1789762765] req@ffff8d25d25ba680 x1876699507693952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762765.564173:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507693952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762765.564179:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762765.564181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762765.564183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507693952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762765.564188:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762765.564201:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762765.564208:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762765/real 1789762765] req@ffff8d25d25bad80 x1876699507694080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762765.564216:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507694080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762765.564222:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762765.564225:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762765.564227:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762765.564242:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507694208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762765.564254:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762765.564262:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762765/real 1789762765] req@ffff8d25d25bbb80 x1876699507694208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762765.564270:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507694208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762765.564276:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762765.564278:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762765.564280:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507694208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762765.564284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507694336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762765.564296:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762765.564303:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762765/real 1789762765] req@ffff8d25d25b9180 x1876699507694336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762765.564311:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507694336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762790 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762765.564317:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762765.564320:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762765.564322:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507694336:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762765.627570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762766.651677:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762767.675583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762768.699716:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762769.723580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762770.683505:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762770.683517:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762770.683522:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762770.683538:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2748 00000100:00080000:0.0:1789762770.683549:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762770.683557:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762770.683599:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507694464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762770.683633:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762770.683636:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762770.683638:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762770.683644:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2748 00000100:00080000:0.0:1789762770.683649:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762770.683655:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762770.683665:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762770.683675:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507694464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762770.683682:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507694592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762770.683696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762770.683699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762770.683701:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762770.683706:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2748 00000100:00080000:0.0:1789762770.683712:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762770.683717:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762770.683720:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762770.683730:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507694720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762770.683739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762770.683742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762770.683744:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762770.683747:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2748 00000100:00080000:0.0:1789762770.683751:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762770.683754:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789762770.683758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762770.683759:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762770.683769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507694848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762770.683776:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762770.683779:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762770.683781:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762770.683784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2748 00000100:00080000:0.0:1789762770.683789:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762770.683792:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762770.683795:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762770.683816:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507694976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762770.683823:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789762770.683828:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762770/real 1789762770] req@ffff8d25d25b8000 x1876699507694464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762770.683829:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762770.683836:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762770.683842:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2748 00000100:00080000:0.0:1789762770.683845:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762770.683848:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762770.683851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507694464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762770.683853:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789762770.683867:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762770.683869:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507695104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762770.683873:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762770.683885:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507694464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762770.683915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762770.683935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762770.683946:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762770/real 1789762770] req@ffff8d25d25ba300 x1876699507694592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762770.683954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507694592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762770.683961:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762770.683964:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762770.683967:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762770.683971:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762770.683986:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762770.683999:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762770/real 1789762770] req@ffff8d25d25b8a80 x1876699507694720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762770.684011:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507694720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762770.684020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762770.684024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762770.684028:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762770.684038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762770.684064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762770.684076:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762770/real 1789762770] req@ffff8d25d25b9180 x1876699507694848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762770.684089:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507694848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762770.684095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762770.684099:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762770.684102:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507694848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762770.684110:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507694976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762770.684141:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762770.684159:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762770/real 1789762770] req@ffff8d25d25bbb80 x1876699507694976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762770.684167:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507694976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762770.684173:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762770.684175:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762770.684177:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507694976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762770.684181:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507695104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762770.684195:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762770.684201:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762770/real 1789762770] req@ffff8d25d25bad80 x1876699507695104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762770.684209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507695104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762795 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762770.684214:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762770.684216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762770.684218:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507695104:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762770.747546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762771.771580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762772.795560:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762773.819554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762774.843611:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762775.803450:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762775.803458:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762775.803463:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762775.803471:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2753 00000100:00080000:0.0:1789762775.803482:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762775.803488:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762775.803531:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507695232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762775.803560:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762775.803563:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762775.803564:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762775.803567:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2753 00000100:00080000:0.0:1789762775.803570:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762775.803572:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762775.803580:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762775.803590:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507695360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762775.803598:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762775.803601:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762775.803603:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762775.803608:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2753 00000100:00080000:0.0:1789762775.803610:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762775.803613:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762775.803618:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762775.803625:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507695488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762775.803634:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762775.803637:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762775.803638:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762775.803641:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2753 00000100:00080000:0.0:1789762775.803644:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762775.803652:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762775.803657:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507695232:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762775.803659:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762775.803679:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507695616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762775.803694:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762775.803699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762775.803701:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762775.803707:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2753 00000100:00080000:0.0:1789762775.803711:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762775.803716:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762775.803723:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762775.803743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507695744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762775.803751:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762775.803755:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762775.803757:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762775.803759:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2753 00000100:00080000:0.0:1789762775.803779:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762775.803784:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762775.803791:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789762775.803807:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762775.803815:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507695872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762775.803891:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762775/real 1789762775] req@ffff8d25d25ba680 x1876699507695232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762775.803926:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507695232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762775.803941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762775.803946:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762775.803951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507695232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762775.803961:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507695360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762775.803990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762775.804005:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762775/real 1789762775] req@ffff8d25d25bb800 x1876699507695360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762775.804020:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507695360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762775.804030:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762775.804034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762775.804038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507695360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762775.804073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507695488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762775.804093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762775.804104:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762775/real 1789762775] req@ffff8d25d25b9f80 x1876699507695488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762775.804118:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507695488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762775.804135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762775.804139:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762775.804160:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507695488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762775.804168:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507695616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762775.804191:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762775.804203:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762775/real 1789762775] req@ffff8d25d25b8380 x1876699507695616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762775.804217:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507695616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762775.804228:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762775.804231:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762775.804235:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507695616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762775.804242:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507695744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762775.804261:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762775.804274:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762775/real 1789762775] req@ffff8d25d25b8700 x1876699507695744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762775.804302:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507695744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762775.804312:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762775.804316:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762775.804321:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507695744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762775.804329:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507695872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762775.804351:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762775.804363:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762775/real 1789762775] req@ffff8d25d25bb100 x1876699507695872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762775.804448:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507695872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762800 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762775.804459:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762775.804463:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762775.804468:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507695872:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762775.867527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762776.891546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762777.915571:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762778.939534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762779.963625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762780.923496:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762780.923507:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762780.923511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762780.923522:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2758 00000100:00080000:0.0:1789762780.923534:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762780.923542:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762780.923595:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507696000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762780.923621:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762780.923624:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762780.923625:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762780.923629:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2758 00000100:00080000:0.0:1789762780.923631:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762780.923634:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762780.923645:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762780.923653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507696128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762780.923661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762780.923663:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762780.923664:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762780.923667:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2758 00000100:00080000:0.0:1789762780.923668:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762780.923671:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762780.923673:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762780.923680:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507696256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762780.923688:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762780.923692:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762780.923693:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762780.923696:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2758 00000100:00080000:0.0:1789762780.923700:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762780.923703:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762780.923708:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762780.923719:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507696000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762780.923720:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507696384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762780.923755:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762780.923758:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762780.923760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762780.923766:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2758 00000100:00080000:0.0:1789762780.923770:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762780.923773:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762780.923779:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762780.923789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507696512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762780.923796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762780.923800:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762780.923801:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789762780.923809:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762780.923810:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2758 00000100:00080000:0.0:1789762780.923821:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762780.923826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762780.923830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762780.923844:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507696640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762780.923872:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762780/real 1789762780] req@ffff8d25d25b9c00 x1876699507696000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762780.923889:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507696000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762780.923899:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762780.923902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762780.923906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507696000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762780.923912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762780.923927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762780.923936:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762780/real 1789762780] req@ffff8d25d25b8e00 x1876699507696128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762780.923944:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507696128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762780.923950:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762780.923952:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762780.923954:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762780.923958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762780.923968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762780.923974:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762780/real 1789762780] req@ffff8d25d25bb480 x1876699507696256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762780.923982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507696256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762780.923987:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762780.923988:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762780.923990:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762780.924011:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762780.924022:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762780.924028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762780/real 1789762780] req@ffff8d25d25bb100 x1876699507696384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762780.924041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507696384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762780.924046:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762780.924048:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762780.924050:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762780.924053:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507696512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762780.924064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762780.924070:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762780/real 1789762780] req@ffff8d25d25b8700 x1876699507696512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762780.924077:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507696512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762780.924082:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762780.924084:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762780.924086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507696512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762780.924089:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507696640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762780.924099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762780.924105:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762780/real 1789762780] req@ffff8d25d25b8380 x1876699507696640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762780.924112:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507696640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762805 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762780.924121:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762780.924124:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762780.924126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507696640:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762780.987516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762782.011704:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762783.035509:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762784.059566:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762785.083508:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762786.043542:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762786.043570:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762786.043574:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.043586:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2763 00000100:00080000:2.0:1789762786.043599:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.043608:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762786.043648:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507696768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762786.043682:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762786.043686:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762786.043714:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.043726:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2763 00000100:00080000:2.0:1789762786.043732:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762786.043751:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507696768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762786.043759:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762786.043781:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762786.043800:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507696896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762786.043813:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762786.043817:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762786.043819:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.043825:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2763 00000100:00080000:2.0:1789762786.043839:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.043843:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762786.043848:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762786.043854:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762786.043860:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507697024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762786.043872:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762786.043876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762786.043878:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.043882:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2763 00000100:00080000:2.0:1789762786.043885:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.043888:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762786.043897:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762786.043906:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507697152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762786.043917:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762786.043923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffff8d25c8023480 x1876699507696768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762786.043925:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762786.043930:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.043948:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2763 00000100:00080000:2.0:1789762786.043953:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.043958:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762786.043965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507696768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762786.043969:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762786.043995:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507697280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762786.044003:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762786.044022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762786.044033:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762786.044036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507696768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762786.044043:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762786.044051:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762786.044057:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2763 00000100:00080000:2.0:1789762786.044062:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762786.044066:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762786.044072:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762786.044080:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762786.044087:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507697408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762786.044109:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762786.044125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffff8d25c8023100 x1876699507696896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762786.044141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507696896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762786.044157:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762786.044160:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762786.044162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507696896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762786.044167:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762786.044179:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762786.044186:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffff8d25c8022d80 x1876699507697024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762786.044194:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507697024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762786.044200:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762786.044201:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762786.044203:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762786.044207:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762786.044219:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762786.044225:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffff8d25c8020380 x1876699507697152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762786.044233:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507697152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762786.044254:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762786.044257:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762786.044259:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762786.044262:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507697280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762786.044273:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762786.044279:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffff8d25c8022300 x1876699507697280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762786.044287:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507697280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762786.044292:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762786.044294:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762786.044296:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507697280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762786.044308:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507697408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762786.044320:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762786.044327:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762786/real 1789762786] req@ffff8d25c8023b80 x1876699507697408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762786.044334:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507697408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762811 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762786.044340:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762786.044341:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762786.044344:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507697408:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762786.107574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762787.131553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762788.155551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762789.179506:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762790.203558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762791.163495:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762791.163503:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762791.163507:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762791.163515:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2768 00000100:00080000:0.0:1789762791.163522:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762791.163528:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762791.163560:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507697536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762791.163582:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762791.163585:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762791.163586:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762791.163591:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2768 00000100:00080000:0.0:1789762791.163596:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762791.163601:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762791.163611:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762791.163613:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507697536:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762791.163632:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507697664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762791.163646:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762791.163661:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762791.163664:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762791.163668:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2768 00000100:00080000:0.0:1789762791.163672:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762791.163675:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762791.163680:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762791.163693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762791.163695:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507697792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762791.163709:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762791.163712:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762791.163715:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762791.163720:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2768 00000100:00080000:0.0:1789762791.163724:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762791.163729:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762791.163735:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762791.163737:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffff8d25d25b9f80 x1876699507697536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762791.163748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507697920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762791.163756:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507697536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762791.163758:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762791.163767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762791.163771:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762791.163776:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762791.163777:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2768 00000100:00080000:0.0:1789762791.163783:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762791.163785:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762791.163790:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762791.163794:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507697536:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762791.163796:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762791.163807:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507698048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762791.163813:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762791.163816:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762791.163817:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762791.163819:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697664:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762791.163824:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2768 00000100:00080000:0.0:1789762791.163827:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762791.163830:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762791.163833:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762791.163842:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507698176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762791.163849:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762791.163866:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffff8d25d25bb800 x1876699507697664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762791.163876:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507697664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762791.163882:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762791.163884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762791.163886:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762791.163890:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762791.163902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762791.163908:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffff8d25d25ba680 x1876699507697792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762791.163915:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507697792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762791.163920:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762791.163922:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762791.163924:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762791.163927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762791.163938:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762791.163943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffff8d25d25bad80 x1876699507697920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762791.163953:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507697920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762791.163962:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762791.163965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762791.163968:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507697920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762791.163974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762791.163994:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762791.164006:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffff8d25d25bbb80 x1876699507698048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762791.164019:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507698048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762791.164024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762791.164026:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762791.164029:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762791.164041:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762791.164055:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762791.164062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762791/real 1789762791] req@ffff8d25d25b9180 x1876699507698176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762791.164069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507698176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762816 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762791.164074:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762791.164076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762791.164077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698176:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762791.227524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762792.251571:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762793.275509:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762794.299505:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762795.323638:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762796.283536:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762796.283564:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762796.283569:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762796.283582:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2773 00000100:00080000:0.0:1789762796.283593:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762796.283630:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762796.283689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507698304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762796.283719:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762796.283723:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762796.283725:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762796.283729:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2773 00000100:00080000:0.0:1789762796.283733:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762796.283739:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762796.283751:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762796.283762:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507698304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762796.283774:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507698432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762796.283793:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762796.283798:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762796.283802:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762796.283808:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2773 00000100:00080000:0.0:1789762796.283815:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762796.283820:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762796.283826:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762796.283840:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507698560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762796.283850:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762796.283853:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762796.283855:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762796.283859:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2773 00000100:00080000:0.0:1789762796.283863:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762796.283867:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762796.283871:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762796.283881:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507698432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762796.283888:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507698688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762796.283903:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762796.283907:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762796.283909:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762796.283913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2773 00000100:00080000:0.0:1789762796.283917:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762796.283919:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507698560:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762796.283925:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762796.283941:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762796.283957:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507698816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762796.283960:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507698688:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1789762796.284010:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762796.284015:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762796.284018:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762796.284025:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2773 00000100:00080000:0.0:1789762796.284031:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762796.284036:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762796.284038:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762796.284051:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762796.284066:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507698944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762796.284139:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698944:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762796.347552:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762797.371516:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762798.395481:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789762799.355764:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789762799.355799:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789762799.355813:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762799.355848:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762799.355869:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762799.355879:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762799.355890:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762799.355899:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789762799.356863:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762799] req@ffff8d25d25bad80 x1876699507698944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762799.356892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507698944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762799.356904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762799.356926:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762799.356930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762799.356939:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762799] req@ffff8d25d25bbb80 x1876699507698816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762799.356951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507698816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762799.356958:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762799.356962:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762799.356965:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507698816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762799.356972:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762799] req@ffff8d25d25b9180 x1876699507698688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762799.356983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507698688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762799.357005:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762799.357008:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762799.357011:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507698688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762799.357017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762799] req@ffff8d25d25ba300 x1876699507698432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762799.357028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507698432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762799.357035:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762799.357038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762799.357041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507698432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762799.357047:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762799] req@ffff8d25d25b8000 x1876699507698560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762799.357065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507698560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762799.357072:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762799.357076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762799.357084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507698560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762799.357092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762796/real 1789762799] req@ffff8d25d25b8a80 x1876699507698304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762799.357104:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507698304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762821 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762799.357162:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762799.357166:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762799.357170:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507698304:192.168.203.141@tcp:250:kworker.0 00010000:00010000:3.0:1789762799.419553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762800.443497:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762801.403496:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762801.403509:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762801.403514:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762801.403529:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2779 00000100:00080000:2.0:1789762801.403541:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762801.403551:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762801.403613:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507699072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762801.403644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762801.403647:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762801.403649:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762801.403653:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2779 00000100:00080000:2.0:1789762801.403656:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762801.403661:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762801.403671:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762801.403682:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507699072:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762801.403685:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507699200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762801.403699:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762801.403701:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762801.403702:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762801.403705:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2779 00000100:00080000:2.0:1789762801.403707:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762801.403709:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762801.403713:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762801.403721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507699328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762801.403727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762801.403729:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762801.403731:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762801.403733:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2779 00000100:00080000:2.0:1789762801.403736:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762801.403739:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762801.403743:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762801.403753:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507699456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762801.403765:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762801.403769:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762801.403771:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762801.403776:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2779 00000800:00000100:3.0:1789762801.403781:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762801.403783:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762801.403791:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762801.403798:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762801.403812:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507699584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762801.403822:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762801.403826:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762801.403829:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762801.403833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2779 00000100:00080000:2.0:1789762801.403838:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762801.403844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762801/real 1789762801] req@ffff8d25c8023800 x1876699507699072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762801.403845:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762801.403855:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762801.403871:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507699072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762801.403872:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507699712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762801.403905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762801.403913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762801.403920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507699072:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762801.403955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762801.403995:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762801.404010:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762801/real 1789762801] req@ffff8d25c8022680 x1876699507699200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762801.404023:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507699200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762801.404033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762801.404038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762801.404042:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762801.404049:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762801.404072:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762801.404086:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762801/real 1789762801] req@ffff8d25c8021180 x1876699507699328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762801.404099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507699328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762801.404109:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762801.404112:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762801.404116:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762801.404123:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762801.404146:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762801.404158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762801/real 1789762801] req@ffff8d25c8020a80 x1876699507699456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762801.404168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507699456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762801.404174:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762801.404177:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762801.404181:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762801.404188:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507699584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762801.404209:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762801.404222:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762801/real 1789762801] req@ffff8d25c8020000 x1876699507699584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762801.404237:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507699584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762801.404244:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762801.404248:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762801.404252:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507699584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762801.404259:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507699712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762801.404411:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762801.404425:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762801/real 1789762801] req@ffff8d25c8022a00 x1876699507699712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762801.404437:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507699712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762826 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762801.404445:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762801.404449:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762801.404452:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507699712:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762801.467630:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762802.491490:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762803.515544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762804.539503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762805.563503:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762806.523493:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762806.523501:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762806.523504:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762806.523514:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2784 00000100:00080000:3.0:1789762806.523522:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762806.523529:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762806.523632:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507699840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762806.523670:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762806.523675:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762806.523678:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762806.523685:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2784 00000100:00080000:3.0:1789762806.523694:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762806.523704:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507699840:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762806.523704:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762806.523724:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762806.523754:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507699968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762806.523768:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789762806.523770:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762806.523812:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762806/real 1789762806] req@ffff8d25d1b08a80 x1876699507699840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762806.523828:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507699840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762806.523837:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762806.523840:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762806.523842:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507699840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762806.523863:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762806.523876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762806.523885:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789762806.523886:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762806.523899:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2784 00000100:00080000:3.0:1789762806.523905:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762806.523907:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762806/real 1789762806] req@ffff8d25d1b0aa00 x1876699507699968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762806.523912:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762806.523920:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762806.523938:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507699968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762806.523946:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762806.523948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762806.523951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507699968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762806.524081:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507700096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762806.524097:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762806.524100:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762806.524103:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762806.524107:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2784 00000100:00080000:3.0:1789762806.524110:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762806.524130:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762806.524156:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762806.524178:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762806/real 1789762806] req@ffff8d25d1b09180 x1876699507700096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762806.524184:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762806.524197:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762806.524215:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507700096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762806.524220:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507700224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762806.524263:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762806.524271:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789762806.524274:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762806.524275:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789762806.524284:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762806.524287:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762806.524301:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2784 00000100:00080000:3.0:1789762806.524309:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762806.524314:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700224:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789762806.524316:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762806.524329:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762806.524344:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507700352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762806.524354:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789762806.524364:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762806.524365:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762806.524369:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762806.524413:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2784 00000100:00080000:3.0:1789762806.524419:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762806.524423:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762806.524430:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762806.524448:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507700480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762806.524449:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762806/real 1789762806] req@ffff8d25d1b0ad80 x1876699507700224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762806.524521:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507700224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762806.524530:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762806.524532:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762806.524548:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762806.524562:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507700352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762806.524580:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762806.524589:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762806/real 1789762806] req@ffff8d25d1b09c00 x1876699507700352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762806.524597:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507700352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762806.524602:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762806.524604:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762806.524606:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507700352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762806.524609:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507700480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762806.524620:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762806.524626:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762806/real 1789762806] req@ffff8d25d1b0b480 x1876699507700480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762806.524633:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507700480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762831 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762806.524638:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762806.524640:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762806.524642:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507700480:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762806.587569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762807.611572:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762808.635559:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762809.659585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762810.683529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762811.643529:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762811.643540:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762811.643544:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762811.643554:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2789 00000100:00080000:2.0:1789762811.643563:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762811.643571:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762811.643612:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507700608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762811.643638:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762811.643641:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762811.643643:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762811.643647:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2789 00000100:00080000:2.0:1789762811.643650:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762811.643654:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762811.643661:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762811.643671:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507700736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762811.643679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762811.643682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762811.643684:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762811.643687:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2789 00000100:00080000:2.0:1789762811.643689:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762811.643691:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762811.643695:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762811.643698:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507700608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762811.643705:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507700864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762811.643713:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762811.643715:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762811.643716:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762811.643718:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2789 00000100:00080000:2.0:1789762811.643720:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762811.643723:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762811.643725:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762811.643733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507700992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762811.643739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762811.643741:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762811.643743:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762811.643746:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2789 00000100:00080000:2.0:1789762811.643748:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762811.643750:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762811.643754:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762811.643757:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762811.643764:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507701120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762811.643775:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762811.643778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762811.643780:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762811.643784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2789 00000100:00080000:2.0:1789762811.643788:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762811.643795:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762811.643796:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762811/real 1789762811] req@ffff8d25c8022a00 x1876699507700608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789762811.643801:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762811.643819:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507701248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762811.643825:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507700608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762811.643844:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762811.643848:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762811.643853:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507700608:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762811.643859:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762811.643878:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762811.643887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762811/real 1789762811] req@ffff8d25c8020000 x1876699507700736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762811.643895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507700736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762811.643901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762811.643903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762811.643905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762811.643922:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762811.643933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762811.643940:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762811/real 1789762811] req@ffff8d25c8020a80 x1876699507700864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762811.643947:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507700864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762811.643957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762811.643960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762811.643962:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762811.643965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762811.643974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762811.643980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762811/real 1789762811] req@ffff8d25c8021180 x1876699507700992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762811.643987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507700992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762811.643993:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762811.643994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762811.643996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507700992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762811.643999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507701120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762811.644009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762811.644015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762811/real 1789762811] req@ffff8d25c8022680 x1876699507701120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762811.644022:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507701120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762811.644027:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762811.644029:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762811.644031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507701120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762811.644035:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507701248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762811.644045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762811.644051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762811/real 1789762811] req@ffff8d25c8023800 x1876699507701248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762811.644058:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507701248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762836 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762811.644063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762811.644065:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762811.644068:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507701248:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762811.707549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762812.731502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762813.755510:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762814.779556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762815.803567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762816.763516:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762816.763524:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762816.763528:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762816.763536:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2794 00000100:00080000:0.0:1789762816.763545:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762816.763551:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762816.763581:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507701376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762816.763605:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762816.763609:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762816.763610:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762816.763613:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2794 00000100:00080000:0.0:1789762816.763616:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762816.763618:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762816.763625:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762816.763636:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507701504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762816.763644:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762816.763647:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762816.763648:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762816.763651:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2794 00000100:00080000:0.0:1789762816.763653:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762816.763663:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762816.763663:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507701376:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762816.763676:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762816.763686:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507701632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762816.763693:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762816.763695:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762816.763697:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762816.763700:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2794 00000100:00080000:0.0:1789762816.763702:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762816.763704:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762816.763708:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762816.763717:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507701760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762816.763724:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762816.763727:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762816.763728:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762816.763730:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2794 00000100:00080000:0.0:1789762816.763742:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762816.763748:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789762816.763752:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762816.763753:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762816.763774:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507701888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762816.763786:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762816.763789:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762816.763790:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762816.763795:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2794 00000100:00080000:0.0:1789762816.763800:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762816.763803:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762816.763808:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762816.763815:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762816/real 1789762816] req@ffff8d25d25ba680 x1876699507701376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762816.763819:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507702016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762816.763895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507701376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762816.763909:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762816.763913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762816.763917:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507701376:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762816.763943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507701504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762816.763966:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762816.763978:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762816/real 1789762816] req@ffff8d25d25bb800 x1876699507701504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762816.763987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507701504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762816.763994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762816.763996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762816.763999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507701504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762816.764003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507701632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762816.764019:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762816.764031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762816/real 1789762816] req@ffff8d25d25b9f80 x1876699507701632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762816.764045:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507701632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762816.764054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762816.764058:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762816.764062:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507701632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762816.764069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507701760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762816.764088:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762816.764096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762816/real 1789762816] req@ffff8d25d25b8380 x1876699507701760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762816.764107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507701760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762816.764116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762816.764120:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762816.764124:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507701760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762816.764131:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507701888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762816.764152:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762816.764163:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762816/real 1789762816] req@ffff8d25d25b8700 x1876699507701888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762816.764171:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507701888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762816.764177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762816.764179:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762816.764181:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507701888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762816.764185:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507702016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762816.764199:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762816.764205:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762816/real 1789762816] req@ffff8d25d25bb100 x1876699507702016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762816.764212:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507702016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762841 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762816.764217:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762816.764219:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762816.764222:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507702016:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762816.827600:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762817.851550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762818.875537:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762819.899513:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762820.923597:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762821.883491:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762821.883499:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762821.883503:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762821.883511:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2799 00000100:00080000:1.0:1789762821.883519:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762821.883524:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762821.883554:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507702144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762821.883575:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762821.883578:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762821.883579:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762821.883583:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2799 00000100:00080000:1.0:1789762821.883585:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762821.883589:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762821.883597:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762821.883607:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507702144:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762821.883608:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507702272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762821.883622:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762821.883624:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762821.883625:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762821.883628:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2799 00000100:00080000:1.0:1789762821.883630:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762821.883633:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762821.883636:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762821.883645:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507702400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762821.883652:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762821.883654:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762821.883655:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762821.883658:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2799 00000100:00080000:1.0:1789762821.883661:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762821.883664:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762821.883667:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762821.883671:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762821.883675:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507702528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762821.883684:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762821.883686:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762821.883688:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762821.883690:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2799 00000100:00080000:1.0:1789762821.883693:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762821.883695:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762821.883698:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762821.883705:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507702656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762821.883710:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762821/real 1789762821] req@ffff8d25c8168000 x1876699507702144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762821.883714:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762821.883723:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762821.883726:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762821.883733:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2799 00000100:00080000:1.0:1789762821.883735:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762821.883740:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762821.883741:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507702144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762821.883743:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762821.883753:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507702784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762821.883755:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762821.883766:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762821.883771:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507702144:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762821.883790:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507702272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762821.883806:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762821.883815:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762821/real 1789762821] req@ffff8d25c8169500 x1876699507702272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762821.883823:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507702272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762821.883828:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762821.883831:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762821.883833:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507702272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762821.883837:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507702400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762821.883847:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762821.883853:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762821/real 1789762821] req@ffff8d25c8168700 x1876699507702400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762821.883861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507702400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762821.883866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762821.883867:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762821.883869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507702400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762821.883873:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507702528:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762821.883883:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762821.883888:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762821/real 1789762821] req@ffff8d25c8168380 x1876699507702528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762821.883895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507702528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762821.883901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762821.883902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762821.883904:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507702528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762821.883908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507702656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762821.883917:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762821.883924:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762821/real 1789762821] req@ffff8d25c8168e00 x1876699507702656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762821.883931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507702656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762821.883936:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762821.883938:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762821.883939:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507702656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762821.883943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507702784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762821.883952:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762821.883958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762821/real 1789762821] req@ffff8d25c8169f80 x1876699507702784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762821.883965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507702784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762846 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762821.883970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762821.883972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762821.883974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507702784:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789762821.947571:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762822.971565:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762823.995626:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762825.019580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762826.043562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762827.003502:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762827.003510:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762827.003513:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762827.003522:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2804 00000100:00080000:0.0:1789762827.003530:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762827.003536:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762827.003568:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507702912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762827.003589:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762827.003591:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762827.003592:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762827.003595:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2804 00000100:00080000:0.0:1789762827.003598:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762827.003600:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762827.003607:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762827.003616:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507703040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762827.003623:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762827.003627:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762827.003630:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762827.003631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507702912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762827.003635:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2804 00000100:00080000:0.0:1789762827.003642:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762827.003645:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762827.003648:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762827.003658:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507703168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762827.003664:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762827.003666:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762827.003668:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762827.003671:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2804 00000100:00080000:0.0:1789762827.003673:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762827.003676:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762827.003679:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762827.003686:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507703296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762827.003693:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762827.003695:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762827.003696:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762827.003699:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2804 00000100:00080000:0.0:1789762827.003703:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762827.003705:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762827.003708:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762827.003713:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762827.003720:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507703424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762827.003727:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762827.003730:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762827.003732:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762827.003734:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2804 00000100:00080000:0.0:1789762827.003736:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762827.003739:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762827.003742:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762827.003755:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507703552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762827.003756:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffff8d25d25bb480 x1876699507702912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.003785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507702912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.003797:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.003801:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762827.003806:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507702912:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762827.003828:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762827.003850:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762827.003863:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffff8d25d25b8e00 x1876699507703040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.003876:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507703040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.003885:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.003889:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762827.003892:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762827.003899:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762827.003919:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762827.003930:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffff8d25d25b9c00 x1876699507703168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.003942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507703168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.003957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.003960:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762827.003964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762827.003970:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762827.003989:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762827.004001:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffff8d25d25bb100 x1876699507703296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.004014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507703296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.004023:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.004027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762827.004031:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762827.004036:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507703424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762827.004056:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762827.004068:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffff8d25d25b8700 x1876699507703424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.004081:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507703424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.004089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.004093:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762827.004097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507703424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762827.004103:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507703552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762827.004122:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762827.004133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762827/real 1789762827] req@ffff8d25d25b8380 x1876699507703552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762827.004145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507703552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762852 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762827.004153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762827.004157:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762827.004160:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507703552:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762827.067542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789762828.091536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762829.115542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762830.139523:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762831.163593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762832.123484:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762832.123496:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762832.123501:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762832.123512:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2809 00000100:00080000:0.0:1789762832.123523:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762832.123529:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762832.123576:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507703680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762832.123605:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762832.123608:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762832.123609:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762832.123613:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2809 00000100:00080000:0.0:1789762832.123616:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762832.123619:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762832.123627:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762832.123639:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507703808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762832.123648:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762832.123651:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762832.123653:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762832.123661:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2809 00000100:00080000:0.0:1789762832.123664:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762832.123665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507703680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762832.123668:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762832.123672:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762832.123686:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507703936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762832.123693:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762832.123697:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762832.123698:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762832.123700:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2809 00000100:00080000:0.0:1789762832.123704:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762832.123706:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762832.123709:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762832.123716:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507704064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762832.123723:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762832.123726:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762832.123727:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762832.123730:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2809 00000100:00080000:0.0:1789762832.123732:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762832.123735:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762832.123738:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762832.123743:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507704192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762832.123750:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762832.123752:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762832.123754:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762832.123756:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2809 00000100:00080000:0.0:1789762832.123759:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762832.123761:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762832.123763:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762832.123773:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762832.123774:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507704320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762832.123851:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffff8d25d25b9f80 x1876699507703680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.123874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507703680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.123887:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.123892:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762832.123897:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507703680:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762832.123931:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762832.123961:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762832.123976:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffff8d25d25bb800 x1876699507703808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.123991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507703808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.124001:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.124005:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762832.124009:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762832.124016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762832.124038:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762832.124051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffff8d25d25ba680 x1876699507703936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.124062:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507703936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.124070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.124074:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762832.124077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507703936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762832.124083:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762832.124105:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762832.124117:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffff8d25d25b8a80 x1876699507704064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.124137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507704064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.124146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.124150:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762832.124153:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762832.124160:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507704192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762832.124182:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762832.124194:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffff8d25d25b8000 x1876699507704192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.124207:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507704192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.124215:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.124218:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762832.124222:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507704192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762832.124229:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507704320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762832.124250:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762832.124261:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762832/real 1789762832] req@ffff8d25d25ba300 x1876699507704320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762832.124273:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507704320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762857 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762832.124288:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762832.124292:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762832.124295:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507704320:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762832.187503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762833.211518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762834.235522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762835.259530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762836.283514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762837.243493:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.243501:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762837.243505:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.243512:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2814 00000100:00080000:1.0:1789762837.243521:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.243527:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762837.243558:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507704448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762837.243581:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.243583:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762837.243585:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.243588:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2814 00000100:00080000:1.0:1789762837.243590:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.243593:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762837.243602:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762837.243610:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507704576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762837.243618:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.243621:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762837.243622:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.243625:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2814 00000100:00080000:1.0:1789762837.243630:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.243639:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762837.243639:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507704448:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789762837.243646:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762837.243662:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507704704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762837.243669:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.243672:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762837.243673:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.243675:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2814 00000100:00080000:1.0:1789762837.243677:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.243679:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762837.243683:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762837.243690:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507704832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762837.243696:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.243698:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762837.243700:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.243702:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2814 00000100:00080000:1.0:1789762837.243707:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.243710:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762837.243713:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762837.243720:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507704960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762837.243726:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762837.243728:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762837.243729:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762837.243733:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2814 00000100:00080000:1.0:1789762837.243735:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762837.243737:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762837.243742:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762837.243746:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762837.243752:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507705088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762837.243818:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffff8d25c816a680 x1876699507704448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.243839:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507704448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.243851:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.243856:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762837.243860:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507704448:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762837.243893:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762837.243919:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762837.243932:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffff8d25c816aa00 x1876699507704576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.243945:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507704576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.243954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.243958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762837.243962:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762837.243968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762837.243987:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762837.243998:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffff8d25c8169f80 x1876699507704704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.244011:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507704704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.244018:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.244022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762837.244026:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762837.244032:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762837.244050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762837.244068:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffff8d25c8168e00 x1876699507704832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.244081:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507704832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.244090:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.244093:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762837.244097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507704832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762837.244103:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507704960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762837.244123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762837.244133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffff8d25c8168380 x1876699507704960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.244145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507704960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.244153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.244156:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762837.244160:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507704960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762837.244166:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507705088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762837.244184:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762837.244195:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762837/real 1789762837] req@ffff8d25c8168700 x1876699507705088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762837.244206:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507705088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762862 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762837.244214:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762837.244218:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762837.244222:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507705088:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762837.307528:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762838.331542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762839.355542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762840.379559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762841.403512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762842.363476:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762842.363484:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762842.363489:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.363500:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2820 00000100:00080000:1.0:1789762842.363510:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.363518:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762842.363554:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507705216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762842.363581:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762842.363586:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762842.363587:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.363592:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2820 00000100:00080000:1.0:1789762842.363596:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.363600:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762842.363609:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762842.363622:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507705344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762842.363634:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762842.363637:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507705216:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762842.363639:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762842.363643:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.363651:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2820 00000100:00080000:1.0:1789762842.363655:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.363659:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762842.363664:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762842.363675:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507705472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762842.363682:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762842.363685:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762842.363686:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.363689:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2820 00000100:00080000:1.0:1789762842.363692:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.363694:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762842.363699:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762842.363706:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507705600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762842.363725:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762842.363725:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762842.363731:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762842.363732:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.363736:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2820 00000100:00080000:1.0:1789762842.363739:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.363742:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762842.363745:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762842.363753:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507705728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762842.363759:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762842.363762:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762842.363763:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762842.363765:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2820 00000100:00080000:1.0:1789762842.363768:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762842.363771:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762842.363775:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762842.363782:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffff8d25c8169500 x1876699507705216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762842.363784:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507705856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762842.363838:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507705216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.363851:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.363856:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762842.363861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507705216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762842.363886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507705344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762842.363918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762842.363947:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffff8d25c8168000 x1876699507705344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.363962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507705344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.363970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.363974:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762842.363977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507705344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762842.363984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507705472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762842.364021:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762842.364038:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffff8d25c8168a80 x1876699507705472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.364054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507705472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.364069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.364074:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762842.364168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507705472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762842.364178:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507705600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762842.364215:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762842.364229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffff8d25c816a300 x1876699507705600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.364241:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507705600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.364250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.364253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762842.364256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507705600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762842.364262:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507705728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762842.364490:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762842.364512:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffff8d25c816ad80 x1876699507705728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.364576:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507705728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.364586:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.364589:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762842.364593:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507705728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762842.364601:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507705856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762842.364624:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762842.364636:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762842/real 1789762842] req@ffff8d25c8169c00 x1876699507705856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762842.364647:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507705856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762867 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762842.364655:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762842.364658:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762842.364662:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507705856:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762842.427539:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762843.451492:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762844.475525:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762845.499513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762846.523596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762847.483474:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762847.483482:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762847.483486:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762847.483495:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2825 00000100:00080000:1.0:1789762847.483502:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762847.483508:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762847.483539:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507705984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762847.483561:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762847.483564:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762847.483565:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762847.483568:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2825 00000100:00080000:1.0:1789762847.483570:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762847.483572:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762847.483580:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762847.483590:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507706112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762847.483599:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762847.483599:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507705984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762847.483608:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762847.483610:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762847.483614:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2825 00000100:00080000:1.0:1789762847.483616:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762847.483618:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762847.483621:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762847.483630:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507706240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762847.483641:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762847.483643:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762847.483645:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762847.483647:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2825 00000100:00080000:1.0:1789762847.483649:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762847.483651:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762847.483655:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762847.483661:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507706368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762847.483668:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762847.483670:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762847.483671:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762847.483673:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2825 00000100:00080000:1.0:1789762847.483676:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789762847.483680:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762847.483683:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762847.483687:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762847.483697:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507706496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762847.483707:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762847.483710:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762847.483712:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762847.483718:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2825 00000100:00080000:1.0:1789762847.483723:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762847.483724:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762847/real 1789762847] req@ffff8d25c816b480 x1876699507705984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762847.483732:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762847.483743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507705984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762847.483750:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762847.483754:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762847.483758:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762847.483761:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507705984:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762847.483765:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507706624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762847.483784:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762847.483804:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762847.483812:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762847/real 1789762847] req@ffff8d25c8169880 x1876699507706112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762847.483820:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507706112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762847.483826:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762847.483828:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762847.483830:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762847.483834:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762847.483844:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762847.483850:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762847/real 1789762847] req@ffff8d25c8169c00 x1876699507706240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762847.483857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507706240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762847.483862:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762847.483864:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762847.483866:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762847.483869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762847.483879:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762847.483884:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762847/real 1789762847] req@ffff8d25c816ad80 x1876699507706368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762847.483896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507706368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762847.483901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762847.483903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762847.483905:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762847.483908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507706496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762847.483917:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762847.483923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762847/real 1789762847] req@ffff8d25c816a300 x1876699507706496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762847.483930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507706496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762847.483935:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762847.483937:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762847.483939:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507706496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762847.483942:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507706624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762847.483951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762847.483957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762847/real 1789762847] req@ffff8d25c8168a80 x1876699507706624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762847.483963:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507706624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762872 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762847.483973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762847.483975:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762847.483977:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507706624:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762847.547513:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762848.571513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762849.595514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762850.619481:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762851.643495:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762852.603454:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762852.603464:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762852.603469:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762852.603480:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2830 00000100:00080000:1.0:1789762852.603491:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762852.603498:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762852.603536:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507706752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762852.603565:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762852.603569:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762852.603571:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762852.603576:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2830 00000100:00080000:1.0:1789762852.603580:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762852.603585:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762852.603595:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762852.603612:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507706880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762852.603612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507706752:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762852.603626:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762852.603629:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762852.603631:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762852.603634:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2830 00000100:00080000:1.0:1789762852.603637:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762852.603639:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762852.603643:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762852.603651:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507707008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762852.603658:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762852.603661:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762852.603662:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762852.603664:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2830 00000100:00080000:1.0:1789762852.603667:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762852.603676:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762852.603680:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762852.603686:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762852.603690:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507707136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762852.603698:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762852.603701:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762852.603702:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762852.603705:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2830 00000100:00080000:1.0:1789762852.603710:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762852.603712:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762852.603716:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762852.603724:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507707264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762852.603730:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762852.603733:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762852.603736:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762852.603738:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762852/real 1789762852] req@ffff8d25c8168000 x1876699507706752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762852.603740:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2830 00000100:00080000:1.0:1789762852.603746:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762852.603753:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762852.603757:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762852.603764:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507707392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762852.603790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507706752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762852.603800:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762852.603803:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762852.603807:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507706752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762852.603826:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762852.603847:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762852.603856:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762852/real 1789762852] req@ffff8d25c8169500 x1876699507706880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762852.603864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507706880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762852.603870:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762852.603872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762852.603874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507706880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762852.603878:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762852.603888:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762852.603895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762852/real 1789762852] req@ffff8d25c8168700 x1876699507707008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762852.603902:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507707008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762852.603907:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762852.603909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762852.603911:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762852.603915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762852.603924:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762852.603930:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762852/real 1789762852] req@ffff8d25c8168380 x1876699507707136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762852.603937:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507707136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762852.603942:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762852.603944:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762852.603946:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762852.603949:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507707264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762852.603959:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762852.603965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762852/real 1789762852] req@ffff8d25c8168e00 x1876699507707264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762852.603972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507707264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762852.603977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762852.603979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762852.603981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507707264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762852.603984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507707392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762852.603993:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762852.603999:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762852/real 1789762852] req@ffff8d25c8169f80 x1876699507707392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762852.604006:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507707392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762877 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762852.604011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762852.604012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762852.604014:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507707392:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762852.667526:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762853.691513:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762854.715494:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762855.739505:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762856.763522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762857.723491:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762857.723502:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762857.723507:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762857.723517:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2835 00000100:00080000:1.0:1789762857.723529:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762857.723537:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762857.723584:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507707520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762857.723607:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762857.723610:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762857.723612:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762857.723616:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2835 00000100:00080000:1.0:1789762857.723621:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762857.723627:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762857.723640:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762857.723640:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507707520:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762857.723661:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507707648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762857.723674:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762857.723678:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762857.723680:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762857.723686:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2835 00000100:00080000:1.0:1789762857.723691:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762857.723695:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762857.723701:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789762857.723717:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762857.723720:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507707776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762857.723730:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762857.723733:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762857.723735:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762857.723739:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2835 00000100:00080000:1.0:1789762857.723745:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762857.723748:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762857.723752:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762857.723763:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507707904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762857.723775:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762857.723778:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789762857.723785:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762857/real 1789762857] req@ffff8d25c816aa00 x1876699507707520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762857.723789:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762857.723799:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2835 00000100:00080000:1.0:1789762857.723805:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762857.723811:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762857.723814:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507707520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762857.723819:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762857.723826:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762857.723831:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762857.723838:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507707520:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762857.723839:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507708032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762857.723850:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762857.723853:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762857.723855:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762857.723858:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2835 00000100:00080000:1.0:1789762857.723861:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762857.723864:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762857.723869:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762857.723869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762857.723884:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507708160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762857.723898:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762857.723911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762857/real 1789762857] req@ffff8d25c816a680 x1876699507707648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762857.723920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507707648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762857.723926:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762857.723929:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762857.723931:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762857.723935:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762857.723946:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762857.723953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762857/real 1789762857] req@ffff8d25c8169f80 x1876699507707776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762857.723961:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507707776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762857.723971:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762857.723973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762857.723975:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762857.723979:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762857.723990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762857.723995:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762857/real 1789762857] req@ffff8d25c8168e00 x1876699507707904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762857.724003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507707904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762857.724008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762857.724010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762857.724012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507707904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762857.724016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762857.724027:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762857.724033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762857/real 1789762857] req@ffff8d25c8168380 x1876699507708032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762857.724041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507708032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762857.724046:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762857.724047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762857.724050:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762857.724062:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762857.724073:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762857.724079:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762857/real 1789762857] req@ffff8d25c8168700 x1876699507708160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762857.724087:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507708160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762882 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762857.724092:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762857.724093:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762857.724095:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708160:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762857.787596:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762858.811605:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762859.835605:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762860.859600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762861.883626:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762862.843560:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762862.843574:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762862.843579:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762862.843592:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2840 00000100:00080000:0.0:1789762862.843606:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762862.843616:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762862.843663:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507708288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762862.843702:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762862.843708:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762862.843711:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762862.843716:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2840 00000100:00080000:0.0:1789762862.843720:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762862.843725:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762862.843737:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762862.843751:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507708416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762862.843765:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762862.843770:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762862.843773:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762862.843778:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2840 00000100:00080000:0.0:1789762862.843782:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762862.843787:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762862.843793:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762862.843812:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507708544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762862.843812:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507708288:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762862.843842:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762862.843846:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762862.843849:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762862.843853:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2840 00000100:00080000:0.0:1789762862.843858:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762862.843862:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762862.843869:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762862.843888:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507708672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762862.843903:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762862.843907:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762862.843909:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762862.843913:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2840 00000100:00080000:0.0:1789762862.843918:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762862.843923:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762862.843928:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762862.843941:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507708800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762862.843953:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762862.843958:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762862.843960:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762862.843969:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2840 00000100:00080000:0.0:1789762862.843978:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762862.843983:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507708416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762862.843992:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762862.844001:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762862.844019:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507708928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762862.844063:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507708544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762862.844098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507708672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762862.844145:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762862.844170:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708928:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762862.907575:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762863.931614:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762864.955603:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789762865.915592:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789762865.915634:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789762865.915651:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762865.915691:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762865.915704:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762865.915716:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762865.915728:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789762865.915741:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789762865.915743:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762862/real 1789762865] req@ffff8d25d25bad80 x1876699507708544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762865.915785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507708544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762865.915799:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762865.915804:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762865.915809:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507708544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762865.915817:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762862/real 1789762865] req@ffff8d25d25ba300 x1876699507708672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762865.915832:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507708672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762865.915841:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762865.915845:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762865.915849:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507708672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762865.915856:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762862/real 1789762865] req@ffff8d25d25b8000 x1876699507708800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762865.915869:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507708800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762865.915877:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762865.915881:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762865.915884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762865.915891:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762862/real 1789762865] req@ffff8d25d25b8a80 x1876699507708928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762865.915905:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507708928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762865.915913:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762865.915917:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762865.915920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507708928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762865.915927:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762862/real 1789762865] req@ffff8d25d25b9180 x1876699507708288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762865.915942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507708288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762865.915951:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762865.915955:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762865.915959:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507708288:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762865.915966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762862/real 1789762865] req@ffff8d25d25bbb80 x1876699507708416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762865.915980:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507708416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762887 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762865.915989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762865.915993:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762865.915997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507708416:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762865.979524:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762867.003529:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762867.963471:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762867.963479:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762867.963482:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762867.963493:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2845 00000100:00080000:2.0:1789762867.963502:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762867.963509:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762867.963545:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507709056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762867.963575:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762867.963579:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762867.963581:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762867.963586:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2845 00000100:00080000:2.0:1789762867.963590:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762867.963594:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762867.963605:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762867.963621:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507709184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762867.963633:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762867.963638:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762867.963640:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507709056:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762867.963641:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762867.963649:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2845 00000100:00080000:2.0:1789762867.963655:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762867.963659:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762867.963665:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762867.963678:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507709312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762867.963688:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762867.963693:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762867.963694:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762867.963698:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2845 00000100:00080000:2.0:1789762867.963702:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762867.963706:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762867.963711:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762867.963718:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762867.963724:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507709440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762867.963736:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762867.963740:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762867.963742:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762867.963746:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2845 00000100:00080000:2.0:1789762867.963751:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762867.963755:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762867.963761:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762867.963766:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762867/real 1789762867] req@ffff8d25c8023b80 x1876699507709056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762867.963777:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507709568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762867.963789:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762867.963792:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762867.963794:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762867.963798:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507709056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762867.963809:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2845 00000100:00080000:3.0:1789762867.963815:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762867.963816:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762867.963826:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789762867.963828:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789762867.963835:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762867.963837:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507709056:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762867.963854:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507709696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762867.963876:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762867.963905:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762867.963919:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762867/real 1789762867] req@ffff8d25c8022300 x1876699507709184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762867.963932:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507709184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762867.963941:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762867.963945:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762867.963949:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762867.963955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762867.963976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762867.963988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762867/real 1789762867] req@ffff8d25c8020380 x1876699507709312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762867.964007:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507709312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762867.964016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762867.964020:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762867.964023:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762867.964029:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762867.964052:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762867.964064:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762867/real 1789762867] req@ffff8d25c8022d80 x1876699507709440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762867.964145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507709440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762867.964154:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762867.964157:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762867.964160:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762867.964165:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507709568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762867.964186:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762867.964205:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762867/real 1789762867] req@ffff8d25c8023100 x1876699507709568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762867.964216:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507709568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762867.964315:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762867.964321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762867.964325:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507709568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762867.964331:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507709696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762867.964354:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762867.964366:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762867/real 1789762867] req@ffff8d25c8023480 x1876699507709696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762867.964419:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507709696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762892 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762867.964427:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762867.964431:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762867.964434:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507709696:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762868.027491:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762869.051541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762870.075538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762871.099512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762872.123497:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762873.083550:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762873.083568:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762873.083575:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762873.083604:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2850 00000100:00080000:3.0:1789762873.083620:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762873.083632:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762873.083684:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507709824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762873.083720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762873.083725:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762873.083727:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762873.083732:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2850 00000100:00080000:3.0:1789762873.083736:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762873.083740:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762873.083752:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762873.083765:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507709952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762873.083776:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762873.083780:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762873.083783:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762873.083787:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2850 00000100:00080000:3.0:1789762873.083790:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762873.083793:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762873.083798:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762873.083808:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507710080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762873.083819:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507709824:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762873.083824:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762873.083832:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762873.083844:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762873.083853:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2850 00000100:00080000:3.0:1789762873.083857:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762873.083861:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762873.083869:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762873.083880:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507710208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762873.083894:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789762873.083895:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762873.083904:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762873.083907:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762873.083914:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2850 00000100:00080000:3.0:1789762873.083920:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762873.083925:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762873.083930:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762873.083945:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507710336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762873.083949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffff8d25d1b0a300 x1876699507709824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762873.083997:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762873.084002:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762873.084004:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762873.084011:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507709824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762873.084016:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2850 00000100:00080000:3.0:1789762873.084027:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762873.084062:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789762873.084063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789762873.084077:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789762873.084079:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762873.084089:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507709824:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762873.084100:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762873.084103:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507710464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762873.084136:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762873.084152:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffff8d25d1b0b100 x1876699507709952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762873.084161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507709952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762873.084167:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762873.084170:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762873.084172:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507709952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762873.084194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762873.084207:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762873.084214:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffff8d25d1b08380 x1876699507710080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762873.084222:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507710080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762873.084227:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762873.084229:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762873.084231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762873.084235:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762873.084245:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762873.084251:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffff8d25d1b09500 x1876699507710208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762873.084258:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507710208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762873.084263:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762873.084265:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762873.084267:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762873.084271:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507710336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762873.084281:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762873.084287:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffff8d25d1b08000 x1876699507710336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762873.084300:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507710336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762873.084305:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762873.084307:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762873.084308:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507710336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762873.084312:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507710464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762873.084322:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762873.084333:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762873/real 1789762873] req@ffff8d25d1b0b480 x1876699507710464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762873.084339:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507710464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762898 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762873.084344:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762873.084346:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762873.084347:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507710464:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762873.147588:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762874.171558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762875.195580:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762876.219489:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762877.243512:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762878.203467:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762878.203474:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762878.203479:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762878.203488:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2855 00000100:00080000:3.0:1789762878.203495:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762878.203501:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762878.203530:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507710592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762878.203548:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762878.203551:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762878.203552:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762878.203555:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2855 00000100:00080000:3.0:1789762878.203558:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762878.203561:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762878.203568:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762878.203580:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507710720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762878.203588:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507710592:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762878.203590:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762878.203597:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762878.203599:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762878.203603:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2855 00000100:00080000:3.0:1789762878.203605:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762878.203607:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762878.203611:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762878.203619:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507710848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762878.203632:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762878.203634:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762878.203636:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762878.203638:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2855 00000100:00080000:3.0:1789762878.203642:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762878.203645:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762878.203650:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789762878.203655:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762878.203665:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507710976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762878.203673:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762878.203676:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762878.203678:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762878.203680:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2855 00000100:00080000:3.0:1789762878.203683:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762878.203686:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762878.203690:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762878.203694:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffff8d25d1b09c00 x1876699507710592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762878.203700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507711104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762878.203713:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762878.203716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789762878.203718:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507710592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762878.203719:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762878.203729:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2855 00000100:00080000:3.0:1789762878.203736:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762878.203743:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789762878.203745:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789762878.203747:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789762878.203752:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762878.203756:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507710592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762878.203761:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507711232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762878.203782:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762878.203799:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762878.203808:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffff8d25d1b0ad80 x1876699507710720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762878.203816:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507710720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762878.203821:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762878.203823:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762878.203825:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762878.203829:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762878.203839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762878.203845:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffff8d25d1b09180 x1876699507710848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762878.203852:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507710848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762878.203858:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762878.203859:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762878.203861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762878.203865:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762878.203875:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762878.203881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffff8d25d1b0aa00 x1876699507710976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762878.203890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507710976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762878.203896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762878.203898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762878.203900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507710976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762878.203903:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507711104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762878.203912:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762878.203918:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffff8d25d1b08a80 x1876699507711104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762878.203925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507711104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762878.203931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762878.203932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762878.203934:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507711104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762878.203937:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507711232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762878.203947:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762878.203952:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762878/real 1789762878] req@ffff8d25d1b0bb80 x1876699507711232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762878.203960:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507711232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762903 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762878.203965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762878.203966:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762878.203968:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507711232:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762878.267521:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762879.291567:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762880.315536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762881.339541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762882.363557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762883.323492:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.323505:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762883.323511:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.323527:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2861 00000100:00080000:3.0:1789762883.323542:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.323554:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762883.323603:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507711360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762883.323632:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.323635:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762883.323637:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.323641:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2861 00000100:00080000:3.0:1789762883.323644:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.323648:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762883.323659:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762883.323669:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507711488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762883.323677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.323681:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762883.323682:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.323685:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2861 00000100:00080000:3.0:1789762883.323686:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.323689:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762883.323696:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507711360:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789762883.323699:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762883.323711:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507711616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762883.323718:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.323721:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762883.323722:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.323725:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2861 00000100:00080000:3.0:1789762883.323727:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.323729:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762883.323733:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762883.323743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507711744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762883.323756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.323759:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762883.323760:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.323762:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2861 00000100:00080000:3.0:1789762883.323766:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.323769:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762883.323774:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762883.323783:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507711872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789762883.323787:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762883.323797:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762883.323801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762883.323802:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762883.323806:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2861 00000100:00080000:3.0:1789762883.323809:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762883.323811:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762883.323814:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762883.323824:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507712000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762883.323846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762883] req@ffff8d25d1b08e00 x1876699507711360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762883.323864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507711360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762883.323882:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762883.323886:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762883.323890:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507711360:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762883.323897:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507711488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762883.323915:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762883.323925:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762883] req@ffff8d25d1b0b800 x1876699507711488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762883.323934:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507711488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762883.323945:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762883.323947:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762883.323950:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507711488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762883.323996:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507711616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762883.324011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762883.324019:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762883] req@ffff8d25d1b0a680 x1876699507711616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762883.324027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507711616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762883.324033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762883.324036:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762883.324038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507711616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762883.324041:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507711744:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762883.324053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762883.324059:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762883] req@ffff8d25d1b08700 x1876699507711744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762883.324067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507711744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762883.324074:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762883.324076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762883.324081:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507711744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762883.324087:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507711872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762883.324108:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762883.324120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762883] req@ffff8d25d1b09880 x1876699507711872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762883.324133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507711872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762883.324139:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762883.324141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762883.324143:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507711872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762883.324147:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507712000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762883.324162:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762883.324170:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762883/real 1789762883] req@ffff8d25d1b0bb80 x1876699507712000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762883.324178:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507712000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762908 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762883.324183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762883.324185:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762883.324187:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507712000:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762883.387545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762884.411501:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762885.435604:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762886.459524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762887.483614:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762888.443581:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762888.443595:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762888.443602:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762888.443617:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2866 00000100:00080000:2.0:1789762888.443632:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762888.443642:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762888.443701:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507712128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762888.443744:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762888.443750:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762888.443753:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762888.443758:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2866 00000100:00080000:2.0:1789762888.443763:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762888.443768:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762888.443782:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762888.443799:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507712256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762888.443826:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762888.443830:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762888.443833:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762888.443842:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2866 00000100:00100000:1.0:1789762888.443845:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507712128:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762888.443848:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762888.443862:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762888.443870:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762888.443890:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507712384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762888.443909:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762888.443917:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762888.443921:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762888.443955:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2866 00000100:00080000:2.0:1789762888.443963:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762888.443972:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762888.443984:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789762888.443994:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762888.444020:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507712512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762888.444045:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762888.444050:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762888.444053:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762888.444058:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2866 00000100:00080000:2.0:1789762888.444064:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762888.444071:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762888.444081:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762888.444098:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762888/real 1789762888] req@ffff8d25c8021880 x1876699507712128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762888.444106:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507712640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762888.444150:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762888.444155:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762888.444158:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762888.444163:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507712128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762888.444169:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2866 00000100:00080000:2.0:1789762888.444181:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762888.444194:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789762888.444200:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789762888.444202:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789762888.444213:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762888.444222:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507712128:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762888.444227:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507712768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762888.444284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507712256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762888.444323:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762888.444342:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762888/real 1789762888] req@ffff8d25c8021c00 x1876699507712256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762888.444359:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507712256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762888.444370:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762888.444374:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762888.444421:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507712256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762888.444430:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507712384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762888.444456:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762888.444471:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762888/real 1789762888] req@ffff8d25c8021500 x1876699507712384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762888.444487:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507712384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762888.444505:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762888.444509:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762888.444513:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507712384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762888.444520:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507712512:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762888.444540:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762888.444552:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762888/real 1789762888] req@ffff8d25c8020e00 x1876699507712512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762888.444580:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507712512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762888.444591:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762888.444595:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762888.444599:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507712512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762888.444605:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507712640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762888.444627:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762888.444640:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762888/real 1789762888] req@ffff8d25c8020700 x1876699507712640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762888.444654:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507712640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762888.444663:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762888.444667:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762888.444671:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507712640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762888.444679:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507712768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762888.444697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762888.444709:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762888/real 1789762888] req@ffff8d25c8023480 x1876699507712768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762888.444722:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507712768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762913 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762888.444732:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762888.444735:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762888.444739:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507712768:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762888.507645:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762889.531623:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762890.555590:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762891.579619:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762892.603532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762893.563478:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762893.563487:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762893.563491:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762893.563500:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2871 00000100:00080000:2.0:1789762893.563509:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762893.563515:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762893.563549:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507712896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762893.563572:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762893.563574:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762893.563576:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762893.563579:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2871 00000100:00080000:2.0:1789762893.563581:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762893.563584:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762893.563591:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762893.563601:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507713024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762893.563611:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762893.563615:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762893.563617:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762893.563621:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2871 00000100:00080000:2.0:1789762893.563625:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762893.563629:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762893.563635:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762893.563636:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507712896:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762893.563651:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507713152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762893.563663:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762893.563666:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762893.563667:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762893.563671:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2871 00000100:00080000:2.0:1789762893.563676:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762893.563679:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762893.563683:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762893.563691:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507713280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762893.563699:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789762893.563704:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762893.563704:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762893.563708:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762893.563713:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2871 00000100:00080000:2.0:1789762893.563716:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762893.563719:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762893.563723:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762893.563730:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507713408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762893.563736:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762893.563738:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762893.563740:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762893.563742:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2871 00000100:00080000:2.0:1789762893.563745:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762893.563747:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762893.563751:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762893.563755:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762893/real 1789762893] req@ffff8d25c8023100 x1876699507712896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762893.563761:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507713536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762893.563794:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507712896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762893.563806:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762893.563811:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762893.563815:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507712896:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762893.563823:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762893.563845:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762893.563859:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762893/real 1789762893] req@ffff8d25c8022d80 x1876699507713024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762893.563871:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507713024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762893.563881:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762893.563884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762893.563903:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762893.563928:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762893.563949:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762893.563960:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762893/real 1789762893] req@ffff8d25c8020380 x1876699507713152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762893.563972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507713152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762893.563981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762893.563984:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762893.563988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762893.563995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762893.564015:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762893.564026:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762893/real 1789762893] req@ffff8d25c8022300 x1876699507713280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762893.564038:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507713280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762893.564046:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762893.564050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762893.564053:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762893.564059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507713408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762893.564078:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762893.564090:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762893/real 1789762893] req@ffff8d25c8023b80 x1876699507713408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762893.564102:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507713408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762893.564110:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762893.564113:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762893.564117:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507713408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762893.564123:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507713536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762893.564142:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762893.564153:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762893/real 1789762893] req@ffff8d25c8023800 x1876699507713536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762893.564165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507713536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762918 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762893.564173:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762893.564177:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762893.564181:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507713536:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762893.627530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762894.651511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762895.675579:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762896.699567:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762897.723641:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762898.683450:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762898.683461:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762898.683465:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762898.683476:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2876 00000100:00080000:2.0:1789762898.683485:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762898.683493:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762898.683535:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507713664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762898.683561:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762898.683564:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762898.683566:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762898.683569:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2876 00000100:00080000:2.0:1789762898.683572:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762898.683574:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762898.683584:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762898.683594:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507713792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762898.683602:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762898.683607:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789762898.683609:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507713664:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762898.683609:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762898.683616:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2876 00000100:00080000:2.0:1789762898.683622:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762898.683624:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762898.683627:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762898.683635:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507713920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762898.683642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762898.683645:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762898.683646:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762898.683648:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2876 00000100:00080000:2.0:1789762898.683651:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762898.683655:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762898.683661:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762898.683674:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507714048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762898.683686:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762898.683690:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762898.683692:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789762898.683698:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789762898.683699:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2876 00000100:00080000:2.0:1789762898.683707:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762898.683712:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762898.683717:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762898.683728:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507714176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762898.683735:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762898.683737:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762898.683739:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762898.683743:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2876 00000100:00080000:2.0:1789762898.683746:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762898.683748:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762898/real 1789762898] req@ffff8d25c8022680 x1876699507713664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762898.683751:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762898.683759:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762898.683773:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507713664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762898.683773:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507714304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762898.683809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762898.683814:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762898.683818:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507713664:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762898.683864:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762898.683885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762898.683895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762898/real 1789762898] req@ffff8d25c8021180 x1876699507713792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762898.683904:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507713792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762898.683910:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762898.683913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762898.683915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762898.683919:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762898.683931:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762898.683945:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762898/real 1789762898] req@ffff8d25c8020a80 x1876699507713920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762898.683953:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507713920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762898.683959:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762898.683961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762898.683964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507713920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762898.683968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762898.683980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762898.683987:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762898/real 1789762898] req@ffff8d25c8020000 x1876699507714048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762898.683994:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507714048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762898.684000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762898.684001:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762898.684003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762898.684007:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507714176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762898.684017:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762898.684023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762898/real 1789762898] req@ffff8d25c8022a00 x1876699507714176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762898.684031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507714176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762898.684037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762898.684039:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762898.684041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507714176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762898.684044:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507714304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762898.684061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762898.684067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762898/real 1789762898] req@ffff8d25c8023800 x1876699507714304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762898.684074:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507714304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762923 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762898.684079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762898.684080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762898.684082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507714304:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762898.747506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762899.771503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762900.795591:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762901.819593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762902.843532:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762903.803518:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762903.803526:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762903.803529:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762903.803538:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2881 00000100:00080000:3.0:1789762903.803548:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762903.803553:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762903.803586:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507714432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762903.803609:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762903.803612:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762903.803613:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762903.803616:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2881 00000100:00080000:3.0:1789762903.803619:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762903.803622:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762903.803629:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762903.803637:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507714560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762903.803645:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762903.803647:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762903.803648:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762903.803651:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2881 00000100:00080000:3.0:1789762903.803653:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762903.803655:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762903.803658:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762903.803665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507714432:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762903.803666:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507714688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762903.803679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762903.803681:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762903.803682:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762903.803685:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2881 00000100:00080000:3.0:1789762903.803688:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762903.803689:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762903.803693:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762903.803700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507714816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762903.803712:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762903.803714:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762903.803715:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762903.803718:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2881 00000100:00080000:3.0:1789762903.803721:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762903.803723:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762903.803726:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789762903.803729:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762903.803737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507714944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762903.803745:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762903.803747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762903.803749:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762903.803752:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2881 00000100:00080000:3.0:1789762903.803754:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762903.803756:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762903.803760:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762903.803769:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762903/real 1789762903] req@ffff8d25d1b08a80 x1876699507714432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762903.803771:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507715072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762903.803828:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507714432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762903.803837:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762903.803840:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762903.803844:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507714432:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762903.803851:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762903.803868:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762903.803877:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762903/real 1789762903] req@ffff8d25d1b0aa00 x1876699507714560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762903.803886:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507714560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762903.803891:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762903.803893:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762903.803896:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762903.803916:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762903.803927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762903.803934:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762903/real 1789762903] req@ffff8d25d1b09180 x1876699507714688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762903.803942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507714688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762903.803947:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762903.803949:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762903.803951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762903.803955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762903.803964:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762903.803971:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762903/real 1789762903] req@ffff8d25d1b0ad80 x1876699507714816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762903.803977:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507714816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762903.803983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762903.803985:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762903.803987:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507714816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762903.803990:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507714944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762903.803999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762903.804005:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762903/real 1789762903] req@ffff8d25d1b09c00 x1876699507714944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762903.804013:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507714944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762903.804017:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762903.804019:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762903.804021:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507714944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762903.804024:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507715072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762903.804033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762903.804040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762903/real 1789762903] req@ffff8d25d1b0b480 x1876699507715072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762903.804050:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507715072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762928 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762903.804058:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762903.804062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762903.804065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507715072:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762903.867523:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762904.891544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762905.915572:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762906.939492:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762907.963521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762908.923518:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762908.923529:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762908.923533:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762908.923545:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2886 00000100:00080000:1.0:1789762908.923556:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762908.923563:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762908.923607:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507715200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762908.923635:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762908.923638:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762908.923639:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762908.923643:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2886 00000100:00080000:1.0:1789762908.923646:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762908.923650:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762908.923660:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762908.923671:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507715200:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762908.923672:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507715328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762908.923687:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762908.923689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762908.923691:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762908.923695:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2886 00000100:00080000:1.0:1789762908.923697:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762908.923700:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762908.923703:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762908.923710:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507715456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762908.923717:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762908.923720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762908.923721:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762908.923724:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2886 00000100:00080000:1.0:1789762908.923727:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762908.923729:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762908.923733:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762908.923740:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507715584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762908.923747:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762908.923749:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762908.923750:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789762908.923755:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762908.923756:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2886 00000100:00080000:1.0:1789762908.923761:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762908.923764:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762908.923767:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762908.923792:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507715712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762908.923799:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762908.923801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762908.923802:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762908.923805:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2886 00000100:00080000:1.0:1789762908.923809:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762908.923812:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762908.923817:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762908.923826:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762908/real 1789762908] req@ffff8d25c8169500 x1876699507715200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762908.923828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507715840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762908.923875:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507715200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762908.923884:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762908.923887:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762908.923891:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507715200:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762908.923913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507715328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762908.923931:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762908.923940:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762908/real 1789762908] req@ffff8d25c8168000 x1876699507715328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762908.923948:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507715328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762908.923954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762908.923956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762908.923958:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507715328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762908.923962:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507715456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762908.923972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762908.923978:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762908/real 1789762908] req@ffff8d25c8168a80 x1876699507715456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762908.923987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507715456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762908.923997:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762908.923999:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762908.924001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507715456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762908.924004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507715584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762908.924015:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762908.924021:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762908/real 1789762908] req@ffff8d25c816a300 x1876699507715584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762908.924028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507715584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762908.924033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762908.924035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762908.924037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507715584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762908.924040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507715712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762908.924050:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762908.924056:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762908/real 1789762908] req@ffff8d25c816ad80 x1876699507715712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762908.924063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507715712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762908.924071:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762908.924074:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762908.924079:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507715712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762908.924085:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507715840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762908.924105:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762908.924117:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762908/real 1789762908] req@ffff8d25c8169c00 x1876699507715840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762908.924130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507715840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762933 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762908.924137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762908.924139:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762908.924142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507715840:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762908.987549:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762910.011576:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762911.035521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762912.059510:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762913.083541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762914.043600:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762914.043614:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762914.043621:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762914.043637:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2891 00000100:00080000:0.0:1789762914.043651:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762914.043662:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762914.043714:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507715968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762914.043798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762914.043803:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762914.043806:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762914.043813:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2891 00000100:00080000:0.0:1789762914.043818:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762914.043823:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762914.043832:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507715968:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789762914.043834:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762914.043882:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507716096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762914.043905:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762914.043911:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762914.043915:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762914.043923:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2891 00000800:00000100:1.0:1789762914.043930:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762914.043937:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762914.043952:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762914.043962:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762914.043980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffff8d25d25ba680 x1876699507715968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762914.043990:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507716224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762914.044014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507715968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762914.044053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762914.044056:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762914.044063:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762914.044070:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:1.0:1789762914.044072:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507715968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762914.044079:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762914.044092:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2891 00000100:00080000:0.0:1789762914.044100:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789762914.044106:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716096:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762914.044113:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762914.044128:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789762914.044146:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762914.044159:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507716352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789762914.044175:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffff8d25d25bb800 x1876699507716096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762914.044188:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762914.044204:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762914.044209:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789762914.044231:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507716096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762914.044245:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2891 00000100:00080000:1.0:1789762914.044247:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762914.044252:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762914.044255:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789762914.044262:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762914.044264:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716096:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789762914.044270:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762914.044277:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762914.044311:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507716480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762914.044332:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762914.044340:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762914.044344:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762914.044354:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2891 00000800:00000100:1.0:1789762914.044363:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762914.044366:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762914.044379:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762914.044404:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762914.044429:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffff8d25d25b9f80 x1876699507716224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762914.044446:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507716224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762914.044447:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507716608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762914.044506:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762914.044516:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762914.044524:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762914.044555:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762914.044589:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762914.044613:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffff8d25d25b8380 x1876699507716352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762914.044628:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507716352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762914.044639:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762914.044643:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762914.044646:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762914.044652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507716480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762914.044669:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762914.044680:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffff8d25d25b8700 x1876699507716480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762914.044691:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507716480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762914.044700:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762914.044703:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762914.044706:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507716480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762914.044711:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507716608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762914.044728:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762914.044761:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762914/real 1789762914] req@ffff8d25d25bb100 x1876699507716608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762914.044790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507716608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762939 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762914.044795:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762914.044798:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762914.044800:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507716608:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762914.107541:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762915.131505:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762916.155518:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762917.179675:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762918.203552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762919.163484:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762919.163492:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762919.163495:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762919.163505:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2896 00000100:00080000:0.0:1789762919.163511:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762919.163517:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762919.163548:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507716736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762919.163569:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762919.163572:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762919.163573:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762919.163576:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2896 00000100:00080000:0.0:1789762919.163579:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762919.163581:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762919.163588:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762919.163595:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507716864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762919.163603:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762919.163606:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762919.163607:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762919.163610:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2896 00000100:00080000:0.0:1789762919.163612:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762919.163615:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762919.163618:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762919.163624:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507716992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762919.163632:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789762919.163633:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507716736:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762919.163635:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762919.163639:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762919.163645:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2896 00000100:00080000:0.0:1789762919.163647:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762919.163656:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762919.163660:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762919.163670:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507717120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762919.163677:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762919.163680:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762919.163681:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762919.163683:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2896 00000100:00080000:0.0:1789762919.163691:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762919.163694:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789762919.163713:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762919.163745:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762919.163753:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffff8d25d25b9c00 x1876699507716736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762919.163757:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507717248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762919.163790:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762919.163792:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762919.163794:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762919.163798:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2896 00000100:00100000:1.0:1789762919.163800:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507716736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762919.163801:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762919.163807:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762919.163813:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762919.163823:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507717376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762919.163824:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762919.163835:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762919.163842:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507716736:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762919.163851:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762919.163873:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762919.163886:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffff8d25d25b8e00 x1876699507716864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762919.163899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507716864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762919.163908:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762919.163911:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762919.163915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762919.163940:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762919.163960:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762919.163971:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffff8d25d25bb480 x1876699507716992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762919.163983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507716992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762919.163991:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762919.163994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762919.163998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507716992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762919.164004:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789762919.164023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762919.164034:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffff8d25d25bb100 x1876699507717120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762919.164046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507717120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762919.164054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762919.164057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762919.164061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789762919.164067:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507717248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762919.164085:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762919.164096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffff8d25d25b8700 x1876699507717248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762919.164108:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507717248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762919.164117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762919.164120:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762919.164123:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507717248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789762919.164129:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507717376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789762919.164147:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762919.164157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762919/real 1789762919] req@ffff8d25d25b8380 x1876699507717376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762919.164169:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507717376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762944 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762919.164177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789762919.164180:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762919.164183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507717376:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762919.227524:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762920.251545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762921.275490:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762922.299693:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762923.323500:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762924.283526:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762924.283534:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762924.283538:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762924.283549:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2901 00000100:00080000:1.0:1789762924.283559:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762924.283566:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762924.283608:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507717504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762924.283639:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762924.283643:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762924.283645:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762924.283649:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2901 00000100:00080000:1.0:1789762924.283652:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762924.283656:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762924.283677:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762924.283692:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507717632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762924.283703:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762924.283707:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762924.283710:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762924.283714:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507717504:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762924.283717:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2901 00000100:00080000:1.0:1789762924.283725:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762924.283729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762924.283733:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762924.283748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507717760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762924.283757:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762924.283760:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762924.283762:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762924.283765:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2901 00000100:00080000:1.0:1789762924.283779:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762924.283786:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762924.283788:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789762924.283797:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762924.283812:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507717888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762924.283822:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762924.283825:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762924.283826:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762924.283830:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2901 00000100:00080000:1.0:1789762924.283833:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762924.283839:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762924.283840:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffff8d25c8169880 x1876699507717504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789762924.283843:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762924.283855:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507718016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762924.283863:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762924.283868:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762924.283868:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507717504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762924.283869:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762924.283877:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2901 00000100:00080000:1.0:1789762924.283882:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762924.283888:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762924.283892:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762924.283894:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789762924.283899:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507718144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762924.283903:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762924.283913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507717504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762924.283952:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762924.283979:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762924.283994:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffff8d25c816b480 x1876699507717632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762924.284009:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507717632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762924.284019:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762924.284023:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762924.284027:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762924.284034:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762924.284053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762924.284065:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffff8d25c8169c00 x1876699507717760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762924.284078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507717760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762924.284086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762924.284089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762924.284093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762924.284100:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762924.284118:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762924.284129:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffff8d25c816ad80 x1876699507717888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762924.284141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507717888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762924.284150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762924.284153:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762924.284157:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507717888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762924.284163:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762924.284183:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762924.284196:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffff8d25c816a300 x1876699507718016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762924.284209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507718016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762924.284218:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762924.284221:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762924.284225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762924.284231:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762924.284254:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762924.284267:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762924/real 1789762924] req@ffff8d25c8168a80 x1876699507718144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762924.284280:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507718144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762949 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762924.284288:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762924.284291:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762924.284296:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718144:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762924.347547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762925.371507:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762926.395576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762927.419615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762928.443563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762929.403537:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762929.403548:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762929.403554:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762929.403565:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2907 00000100:00080000:1.0:1789762929.403578:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762929.403586:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762929.403702:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507718272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762929.403730:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762929.403734:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762929.403736:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762929.403739:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2907 00000100:00080000:1.0:1789762929.403743:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762929.403746:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762929.403756:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762929.403767:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507718400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762929.403776:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762929.403779:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762929.403781:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762929.403784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2907 00000100:00080000:1.0:1789762929.403787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762929.403791:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762929.403794:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762929.403801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507718528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762929.403814:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762929.403817:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762929.403820:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762929.403830:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507718272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762929.403834:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2907 00000100:00080000:1.0:1789762929.403845:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762929.403850:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762929.403853:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762929.403864:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507718656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762929.403872:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762929.403875:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762929.403876:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762929.403879:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2907 00000100:00080000:1.0:1789762929.403881:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762929.403885:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762929.403888:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762929.403895:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507718784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762929.403902:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762929.403905:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762929.403906:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762929.403909:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2907 00000100:00080000:1.0:1789762929.403915:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762929.403918:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762929.403922:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762929.403930:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507718912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762929.403983:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507718400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762929.404055:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507718528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762929.404085:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507718656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762929.404132:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762929.404165:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718912:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762929.467606:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762930.491702:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762931.515568:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789762932.475640:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789762932.475677:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789762932.475689:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762932.475723:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762932.475733:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762932.475742:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762932.475751:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789762932.475761:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789762932.475785:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762929/real 1789762932] req@ffff8d25c8168700 x1876699507718528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762932.475826:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507718528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762932.475838:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762932.475843:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762932.475848:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507718528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762932.475856:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762929/real 1789762932] req@ffff8d25c8168380 x1876699507718656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762932.475870:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507718656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762932.475878:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762932.475882:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762932.475886:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507718656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762932.475893:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762929/real 1789762932] req@ffff8d25c8168e00 x1876699507718784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762932.475905:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507718784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762932.475914:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762932.475917:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762932.475921:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762932.475928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762929/real 1789762932] req@ffff8d25c8169f80 x1876699507718912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762932.475943:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507718912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762932.475952:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762932.475956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762932.475960:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507718912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762932.475968:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762929/real 1789762932] req@ffff8d25c8168000 x1876699507718272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762932.475982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507718272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762932.475990:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762932.475994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762932.475998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507718272:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762932.476005:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762929/real 1789762932] req@ffff8d25c8169500 x1876699507718400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762932.476018:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507718400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762954 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762932.476026:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762932.476030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762932.476034:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507718400:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762932.539498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762933.563677:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762934.523545:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762934.523557:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762934.523563:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762934.523576:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2912 00000100:00080000:2.0:1789762934.523612:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762934.523623:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762934.523669:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507719040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762934.523709:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762934.523714:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762934.523717:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762934.523730:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2912 00000100:00080000:2.0:1789762934.523735:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762934.523740:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762934.523752:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762934.523770:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507719168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762934.523786:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762934.523791:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762934.523796:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762934.523800:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507719040:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762934.523807:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2912 00000100:00080000:2.0:1789762934.523814:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762934.523824:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762934.523837:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762934.523866:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507719296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762934.523885:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762934.523892:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762934.523896:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762934.523905:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2912 00000100:00080000:2.0:1789762934.523913:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762934.523926:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762934.523929:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762934.523944:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762934.523974:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507719424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762934.523996:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762934.524002:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762934.524006:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762934.524015:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762934/real 1789762934] req@ffff8d25c8023b80 x1876699507719040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762934.524020:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2912 00000100:00080000:2.0:1789762934.524036:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762934.524052:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762934.524067:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762934.524079:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507719040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762934.524097:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507719552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762934.524111:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762934.524131:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762934.524133:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789762934.524149:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762934.524152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507719040:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762934.524158:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762934.524168:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2912 00000100:00080000:2.0:1789762934.524186:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762934.524194:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762934.524205:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762934.524229:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507719680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762934.524248:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762934.524310:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762934.524338:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762934/real 1789762934] req@ffff8d25c8022300 x1876699507719168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762934.524362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507719168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762934.524447:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762934.524456:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762934.524463:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762934.524475:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762934.524521:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762934.524547:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762934/real 1789762934] req@ffff8d25c8020380 x1876699507719296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762934.524573:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507719296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762934.524618:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762934.524626:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762934.524632:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762934.524646:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762934.524691:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762934.525487:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762934/real 1789762934] req@ffff8d25c8022d80 x1876699507719424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762934.525680:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507719424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762934.525703:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762934.525708:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762934.525715:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762934.525728:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507719552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762934.525781:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762934.525806:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762934/real 1789762934] req@ffff8d25c8023100 x1876699507719552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762934.525831:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507719552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762934.525848:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762934.525854:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762934.525861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507719552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762934.525903:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507719680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762934.525946:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762934.525967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762934/real 1789762934] req@ffff8d25c8023480 x1876699507719680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762934.525982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507719680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762959 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762934.525992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762934.525996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762934.526001:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507719680:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762934.587608:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762935.611636:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762936.635627:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762937.659531:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762938.683507:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762939.643495:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762939.643504:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762939.643508:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762939.643516:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2917 00000100:00080000:1.0:1789762939.643525:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762939.643531:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762939.643580:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507719808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762939.643603:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762939.643606:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762939.643607:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762939.643610:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2917 00000100:00080000:1.0:1789762939.643613:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762939.643616:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762939.643625:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762939.643635:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507719936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762939.643644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762939.643648:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762939.643649:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762939.643653:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2917 00000100:00080000:1.0:1789762939.643655:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762939.643664:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762939.643668:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507719808:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789762939.643669:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762939.643686:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507720064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762939.643696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762939.643699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762939.643700:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762939.643702:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2917 00000100:00080000:1.0:1789762939.643706:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762939.643709:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762939.643712:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762939.643721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507720192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762939.643730:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762939.643732:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762939.643733:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762939.643735:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2917 00000100:00080000:1.0:1789762939.643740:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762939.643743:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762939.643747:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762939.643755:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507720320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762939.643764:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762939.643777:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762939.643779:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762939.643782:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762939.643788:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2917 00000100:00080000:1.0:1789762939.643799:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762939.643802:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762939.643805:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762939.643816:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507720448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762939.643825:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762939/real 1789762939] req@ffff8d25c816a680 x1876699507719808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762939.643847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507719808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762939.643857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762939.643861:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762939.643864:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507719808:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762939.643870:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762939.643888:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762939.643897:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762939/real 1789762939] req@ffff8d25c816aa00 x1876699507719936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762939.643906:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507719936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762939.643911:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762939.643913:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762939.643915:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507719936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762939.643943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762939.643967:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762939.643979:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762939/real 1789762939] req@ffff8d25c8169500 x1876699507720064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762939.643993:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507720064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762939.644002:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762939.644006:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762939.644010:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762939.644017:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762939.644041:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762939.644053:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762939/real 1789762939] req@ffff8d25c8168000 x1876699507720192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762939.644067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507720192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762939.644077:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762939.644080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762939.644084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762939.644091:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507720320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762939.644123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762939.644136:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762939/real 1789762939] req@ffff8d25c8169f80 x1876699507720320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762939.644158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507720320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762939.644167:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762939.644170:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762939.644174:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507720320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762939.644181:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507720448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762939.644211:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762939.644223:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762939/real 1789762939] req@ffff8d25c8168e00 x1876699507720448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762939.644237:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507720448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762964 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762939.644246:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762939.644249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762939.644252:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507720448:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762939.707489:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762940.731503:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762941.755573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762942.779548:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762943.803497:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762944.763437:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762944.763445:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762944.763449:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762944.763458:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2922 00000100:00080000:1.0:1789762944.763466:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762944.763471:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762944.763503:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507720576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762944.763537:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762944.763540:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762944.763541:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762944.763544:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2922 00000100:00080000:1.0:1789762944.763548:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762944.763551:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762944.763559:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762944.763570:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507720704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762944.763578:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762944.763582:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762944.763584:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507720576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762944.763585:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762944.763592:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2922 00000100:00080000:1.0:1789762944.763595:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762944.763598:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762944.763601:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762944.763609:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507720832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762944.763615:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762944.763618:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762944.763619:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762944.763622:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2922 00000100:00080000:1.0:1789762944.763625:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762944.763627:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762944.763630:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762944.763637:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507720960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762944.763649:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762944.763655:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762944.763658:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762944.763660:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762944.763663:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2922 00000100:00080000:1.0:1789762944.763668:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762944.763671:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762944.763674:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762944.763682:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507721088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762944.763689:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762944/real 1789762944] req@ffff8d25c8168380 x1876699507720576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762944.763691:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762944.763697:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762944.763699:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762944.763706:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2922 00000100:00080000:1.0:1789762944.763711:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762944.763714:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762944.763717:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762944.763720:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507720576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762944.763727:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507721216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762944.763733:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762944.763740:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762944.763744:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507720576:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762944.763762:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762944.763777:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762944.763786:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762944/real 1789762944] req@ffff8d25c8168700 x1876699507720704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762944.763794:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507720704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762944.763799:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762944.763802:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762944.763804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762944.763807:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762944.763817:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762944.763823:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762944/real 1789762944] req@ffff8d25c8168a80 x1876699507720832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762944.763830:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507720832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762944.763836:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762944.763837:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762944.763839:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762944.763843:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762944.763853:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762944.763859:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762944/real 1789762944] req@ffff8d25c816a300 x1876699507720960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762944.763866:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507720960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762944.763872:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762944.763873:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762944.763875:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507720960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762944.763878:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762944.763888:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762944.763894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762944/real 1789762944] req@ffff8d25c816ad80 x1876699507721088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762944.763901:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507721088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762944.763906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762944.763908:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762944.763909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762944.763913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762944.763922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762944.763927:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762944/real 1789762944] req@ffff8d25c8169c00 x1876699507721216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762944.763935:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507721216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762969 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762944.763940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762944.763942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762944.763944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721216:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762944.827504:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762945.851535:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762946.875553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762947.899574:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762948.923509:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762949.883462:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762949.883482:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762949.883487:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762949.883498:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2927 00000100:00080000:1.0:1789762949.883508:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762949.883517:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762949.883554:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507721344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762949.883579:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762949.883582:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762949.883585:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762949.883589:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2927 00000100:00080000:1.0:1789762949.883592:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762949.883597:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762949.883605:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762949.883611:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507721344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762949.883621:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507721472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762949.883631:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762949.883634:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762949.883636:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762949.883639:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2927 00000100:00080000:1.0:1789762949.883642:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762949.883645:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762949.883648:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762949.883657:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507721600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762949.883664:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762949.883667:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762949.883669:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762949.883672:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789762949.883676:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2927 00000100:00080000:1.0:1789762949.883680:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762949.883683:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762949.883687:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762949.883701:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507721728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762949.883710:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762949.883712:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762949.883714:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762949.883717:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2927 00000100:00080000:1.0:1789762949.883720:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762949.883725:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762949/real 1789762949] req@ffff8d25c816b480 x1876699507721344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762949.883725:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762949.883730:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762949.883742:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507721856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762949.883757:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762949.883760:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762949.883761:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789762949.883766:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507721344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762949.883766:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2927 00000100:00080000:1.0:1789762949.883768:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762949.883775:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762949.883782:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789762949.883783:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762949.883789:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789762949.883792:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507721984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762949.883796:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507721344:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762949.883834:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507721472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762949.883856:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762949.883870:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762949/real 1789762949] req@ffff8d25c8169880 x1876699507721472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762949.883883:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507721472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762949.883892:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762949.883896:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762949.883900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507721472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762949.883907:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507721600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762949.883928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762949.883937:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762949/real 1789762949] req@ffff8d25c8169c00 x1876699507721600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762949.883956:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507721600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762949.883966:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762949.883968:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762949.883978:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507721600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762949.883982:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507721728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762949.883996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762949.884002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762949/real 1789762949] req@ffff8d25c816ad80 x1876699507721728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762949.884010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507721728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762949.884016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762949.884018:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762949.884021:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507721728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762949.884031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762949.884045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762949.884057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762949/real 1789762949] req@ffff8d25c816a300 x1876699507721856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762949.884070:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507721856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762949.884079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762949.884083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762949.884088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762949.884095:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762949.884123:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762949.884133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762949/real 1789762949] req@ffff8d25c8168a80 x1876699507721984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762949.884141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507721984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762974 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762949.884147:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762949.884150:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762949.884152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507721984:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762949.947549:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762950.971509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762951.995593:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762953.019513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762954.043532:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762955.003568:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762955.003582:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762955.003589:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762955.003602:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2932 00000100:00080000:3.0:1789762955.003615:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762955.003626:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762955.003669:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507722112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762955.003696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762955.003700:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762955.003702:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762955.003707:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2932 00000100:00080000:3.0:1789762955.003710:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762955.003714:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762955.003724:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762955.003737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507722240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762955.003748:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762955.003752:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762955.003754:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762955.003757:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2932 00000100:00080000:3.0:1789762955.003760:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762955.003767:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762955.003770:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507722112:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789762955.003772:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762955.003789:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507722368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762955.003798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762955.003801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762955.003803:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762955.003806:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2932 00000100:00080000:3.0:1789762955.003809:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762955.003813:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762955.003818:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762955.003828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507722496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762955.003839:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762955.003842:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762955.003844:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789762955.003852:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789762955.003857:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2932 00000100:00080000:3.0:1789762955.003865:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762955.003870:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762955.003875:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762955.003887:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507722624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762955.003897:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762955.003903:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffff8d25d1b08000 x1876699507722112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762955.003903:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762955.003905:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762955.003920:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2932 00000100:00080000:3.0:1789762955.003928:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762955.003933:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762955.003938:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762955.003939:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507722112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762955.003954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789762955.003955:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507722752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762955.003963:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762955.003974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507722112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762955.003989:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507722240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762955.004015:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762955.004028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffff8d25d1b09500 x1876699507722240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762955.004040:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507722240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762955.004049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762955.004052:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762955.004055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507722240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762955.004081:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507722368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762955.004097:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762955.004109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffff8d25d1b08380 x1876699507722368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762955.004142:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507722368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762955.004154:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762955.004160:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762955.004165:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507722368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762955.004176:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507722496:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762955.004210:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762955.004230:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffff8d25d1b0b100 x1876699507722496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762955.004250:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507722496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762955.004263:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762955.004268:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762955.004274:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507722496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762955.004284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507722624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762955.004314:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762955.004330:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffff8d25d1b0a300 x1876699507722624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762955.004346:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507722624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762955.004358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762955.004362:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762955.004367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507722624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762955.004375:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507722752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762955.004467:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762955.004485:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762955/real 1789762955] req@ffff8d25d1b0b480 x1876699507722752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762955.004503:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507722752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762980 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762955.004514:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762955.004519:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762955.004524:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507722752:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762955.067530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762956.091526:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762957.115573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762958.139515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762959.163529:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762960.123558:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762960.123569:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762960.123573:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762960.123584:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2937 00000100:00080000:1.0:1789762960.123596:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762960.123603:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762960.123649:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507722880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762960.123676:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762960.123679:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762960.123681:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762960.123684:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2937 00000100:00080000:1.0:1789762960.123687:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762960.123690:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762960.123699:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762960.123709:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507723008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762960.123717:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762960.123720:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762960.123721:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762960.123724:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2937 00000100:00080000:1.0:1789762960.123727:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762960.123730:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762960.123734:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762960.123743:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507722880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789762960.123744:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507723136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762960.123770:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762960.123773:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762960.123774:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762960.123777:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2937 00000100:00080000:1.0:1789762960.123779:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762960.123781:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762960.123786:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789762960.123794:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507723264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762960.123800:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762960.123803:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762960.123804:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762960.123809:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2937 00000100:00080000:1.0:1789762960.123822:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762960.123826:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762960.123830:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789762960.123838:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762960.123843:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507723392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762960.123858:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762960.123860:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762960.123862:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762960.123865:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2937 00000100:00080000:1.0:1789762960.123868:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762960.123871:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762960.123874:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762960.123883:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507723520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789762960.123894:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffff8d25c8168700 x1876699507722880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762960.123916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507722880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762960.123927:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762960.123932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762960.123936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507722880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762960.123945:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762960.123968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762960.123981:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffff8d25c8168380 x1876699507723008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762960.123993:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507723008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762960.124002:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762960.124005:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762960.124008:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762960.124032:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762960.124049:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762960.124060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffff8d25c8168e00 x1876699507723136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762960.124071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507723136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762960.124079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762960.124082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762960.124085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762960.124092:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762960.124109:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762960.124118:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffff8d25c8169f80 x1876699507723264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762960.124130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507723264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762960.124138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762960.124141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762960.124145:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762960.124151:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507723392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762960.124170:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762960.124181:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffff8d25c8168000 x1876699507723392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762960.124193:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507723392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762960.124201:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762960.124205:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762960.124208:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507723392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762960.124216:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507723520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762960.124233:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762960.124243:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762960/real 1789762960] req@ffff8d25c8169500 x1876699507723520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762960.124254:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507723520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762985 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762960.124262:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762960.124266:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762960.124270:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507723520:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762960.187535:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762961.211562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762962.235520:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762963.259538:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762964.283528:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789762965.243460:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762965.243468:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789762965.243472:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762965.243480:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2942 00000100:00080000:1.0:1789762965.243488:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762965.243494:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789762965.243526:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507723648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762965.243548:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762965.243551:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762965.243552:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762965.243555:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2942 00000100:00080000:1.0:1789762965.243558:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762965.243560:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762965.243570:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762965.243579:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507723776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762965.243588:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789762965.243589:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507723648:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789762965.243592:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762965.243596:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762965.243600:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2942 00000100:00080000:1.0:1789762965.243602:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762965.243605:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762965.243608:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762965.243618:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507723904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762965.243624:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762965.243626:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789762965.243628:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762965.243631:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2942 00000100:00080000:1.0:1789762965.243641:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762965.243643:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762965.243646:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789762965.243649:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789762965.243657:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507724032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762965.243666:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762965.243668:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789762965.243669:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762965.243671:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2942 00000100:00080000:1.0:1789762965.243674:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789762965.243677:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762965.243680:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762965.243687:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffff8d25c816aa00 x1876699507723648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789762965.243690:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507724160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789762965.243716:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789762965.243719:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789762965.243720:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789762965.243723:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2942 00000100:00080000:1.0:1789762965.243726:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789762965.243728:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507723648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789762965.243730:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789762965.243735:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789762965.243745:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507724288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762965.243750:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762965.243758:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762965.243765:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507723648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762965.243791:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762965.243810:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762965.243820:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffff8d25c816a680 x1876699507723776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762965.243829:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507723776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762965.243835:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762965.243837:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762965.243840:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762965.243843:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762965.243854:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762965.243860:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffff8d25c8169500 x1876699507723904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762965.243868:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507723904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762965.243873:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762965.243875:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762965.243876:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507723904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762965.243880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789762965.243891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762965.243897:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffff8d25c8168000 x1876699507724032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762965.243904:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507724032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762965.243909:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762965.243911:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762965.243913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789762965.243916:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507724160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762965.243926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762965.243932:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffff8d25c8169f80 x1876699507724160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762965.243940:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507724160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762965.243944:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762965.243946:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762965.243948:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507724160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789762965.243951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507724288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789762965.243961:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762965.243966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762965/real 1789762965] req@ffff8d25c8168e00 x1876699507724288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762965.243973:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507724288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762990 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762965.243978:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789762965.243980:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789762965.243982:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507724288:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762965.307536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762966.331521:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762967.355497:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762968.379553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762969.403509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762970.363504:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762970.363512:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762970.363516:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762970.363524:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2948 00000100:00080000:0.0:1789762970.363532:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762970.363538:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762970.363570:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507724416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762970.363593:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762970.363596:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762970.363597:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762970.363600:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2948 00000100:00080000:0.0:1789762970.363603:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762970.363606:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762970.363613:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762970.363622:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507724544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762970.363630:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507724416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762970.363633:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762970.363639:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762970.363642:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762970.363646:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2948 00000100:00080000:0.0:1789762970.363648:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762970.363650:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762970.363653:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762970.363662:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507724672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762970.363673:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762970.363676:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762970.363677:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762970.363680:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2948 00000100:00080000:0.0:1789762970.363683:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762970.363686:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762970.363689:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789762970.363697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789762970.363699:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507724800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762970.363708:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762970.363711:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762970.363712:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762970.363715:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2948 00000100:00080000:0.0:1789762970.363717:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762970.363719:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762970.363722:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762970.363732:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507724928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762970.363738:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffff8d25d25b9f80 x1876699507724416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762970.363741:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762970.363749:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762970.363753:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762970.363761:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2948 00000100:00080000:0.0:1789762970.363766:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762970.363772:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762970.363776:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762970.363781:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507724416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762970.363785:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507725056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762970.363801:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.363806:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762970.363810:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507724416:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762970.363830:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762970.363845:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762970.363854:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffff8d25d25bb800 x1876699507724544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.363862:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507724544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.363868:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.363870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762970.363872:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762970.363875:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762970.363886:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762970.363892:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffff8d25d25ba680 x1876699507724672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.363899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507724672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.363909:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.363910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762970.363912:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762970.363916:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762970.363926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762970.363931:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffff8d25d25bbb80 x1876699507724800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.363939:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507724800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.363951:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.363953:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762970.363955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507724800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762970.363959:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507724928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762970.363970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762970.363977:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffff8d25d25b9180 x1876699507724928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.363985:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507724928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.363989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.363991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762970.363994:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507724928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762970.363997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507725056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762970.364007:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762970.364013:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762970/real 1789762970] req@ffff8d25d25b8a80 x1876699507725056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762970.364020:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507725056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789762995 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762970.364025:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762970.364027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762970.364029:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507725056:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762970.427483:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762971.451732:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762972.475535:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762973.499575:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762974.523591:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762975.483586:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762975.483601:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762975.483608:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762975.483623:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2953 00000100:00080000:0.0:1789762975.483640:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762975.483651:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762975.483712:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507725184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762975.483753:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762975.483759:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762975.483762:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762975.483768:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2953 00000100:00080000:0.0:1789762975.483773:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762975.483781:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762975.483795:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762975.483812:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507725312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762975.483826:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507725184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762975.483833:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762975.483840:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762975.483842:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762975.483849:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2953 00000100:00080000:0.0:1789762975.483854:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762975.483860:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762975.483866:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762975.483881:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507725440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762975.483893:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762975.483899:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789762975.483908:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762975.483909:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762975.483933:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2953 00000100:00080000:0.0:1789762975.483942:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762975.483947:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762975.483955:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762975.483978:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507725568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762975.483980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762975/real 1789762975] req@ffff8d25d25b8000 x1876699507725184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762975.484029:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762975.484034:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762975.484036:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762975.484041:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2953 00000100:00100000:3.0:1789762975.484049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507725184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762975.484052:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762975.484068:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789762975.484072:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762975.484079:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789762975.484089:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762975.484093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507725184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762975.484112:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507725696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762975.484129:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789762975.484131:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507725312:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762975.484152:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762975.484157:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762975.484164:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2953 00000100:00080000:0.0:1789762975.484171:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762975.484176:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762975.484187:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789762975.484189:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762975.484208:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762975/real 1789762975] req@ffff8d25d25ba300 x1876699507725312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762975.484211:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507725824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762975.484311:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507725312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762975.484321:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762975.484325:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762975.484328:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507725312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762975.484336:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507725440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762975.484357:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762975.484370:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762975/real 1789762975] req@ffff8d25d25bad80 x1876699507725440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762975.484419:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507725440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762975.484427:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762975.484434:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762975.484438:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507725440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762975.484444:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507725568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762975.484462:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762975.484481:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762975/real 1789762975] req@ffff8d25d25b8a80 x1876699507725568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762975.484495:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507725568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762975.484505:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762975.484507:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762975.484509:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507725568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762975.484522:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507725696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762975.484534:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762975.484541:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762975/real 1789762975] req@ffff8d25d25b9180 x1876699507725696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762975.484549:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507725696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762975.484554:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762975.484556:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762975.484558:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507725696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762975.484562:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507725824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762975.484572:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762975.484578:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762975/real 1789762975] req@ffff8d25d25bbb80 x1876699507725824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762975.484585:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507725824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763000 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762975.484594:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762975.484596:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762975.484598:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507725824:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762975.547596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762976.571507:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762977.595550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762978.619584:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762979.643589:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789762980.603642:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762980.603656:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789762980.603661:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762980.603676:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2958 00000100:00080000:0.0:1789762980.603690:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762980.603700:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789762980.603759:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507725952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762980.603842:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507725952:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789762980.603908:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762980.603922:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762980.603929:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762980.603954:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2958 00000800:00000100:3.0:1789762980.603964:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762980.603970:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762980.603983:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762980.604000:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762980.604033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762980/real 1789762980] req@ffff8d25d25ba680 x1876699507725952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789762980.604049:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507726080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762980.604097:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762980.604104:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762980.604109:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762980.604111:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507725952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789762980.604126:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2958 00000100:00080000:0.0:1789762980.604140:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762980.604155:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789762980.604159:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789762980.604163:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789762980.604170:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762980.604182:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507725952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789762980.604191:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507726208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762980.604209:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762980.604213:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789762980.604216:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762980.604222:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2958 00000100:00080000:0.0:1789762980.604228:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789762980.604237:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726080:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762980.604269:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762980.604280:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789762980.604314:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507726336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789762980.604330:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789762980.604338:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789762980.604340:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789762980.604348:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789762980.604357:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2958 00000100:00080000:0.0:1789762980.604365:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762980.604373:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762980.604378:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762980/real 1789762980] req@ffff8d25d25bb800 x1876699507726080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789762980.604437:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762980.604438:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507726080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762980.604464:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789762980.604466:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507726464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762980.604488:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762980.604513:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726080:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762980.604514:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789762980.604539:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789762980.604544:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789762980.604554:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789762980.604561:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2958 00000100:00080000:0.0:1789762980.604576:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789762980.604582:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789762980.604590:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789762980.604618:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507726592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789762980.604630:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762980.604648:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762980/real 1789762980] req@ffff8d25d25b9f80 x1876699507726208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762980.604712:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507726208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762980.604742:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762980.604748:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762980.604752:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762980.604776:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762980.604806:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762980.604819:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762980/real 1789762980] req@ffff8d25d25b8380 x1876699507726336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762980.604831:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507726336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762980.604839:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762980.604842:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762980.604844:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762980.604851:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507726464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762980.604868:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762980.604878:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762980/real 1789762980] req@ffff8d25d25b8700 x1876699507726464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762980.604889:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507726464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762980.604897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762980.604916:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762980.604919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507726464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762980.604924:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507726592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762980.604940:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762980.604948:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762980/real 1789762980] req@ffff8d25d25bb100 x1876699507726592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762980.604958:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507726592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763005 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762980.604965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762980.604967:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762980.604970:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507726592:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762980.667613:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762981.691613:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762982.715621:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762983.739594:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762984.763619:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762985.723463:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762985.723470:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762985.723474:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762985.723483:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2963 00000100:00080000:3.0:1789762985.723491:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762985.723498:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762985.723527:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507726720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762985.723545:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762985.723548:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762985.723550:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762985.723555:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2963 00000100:00080000:3.0:1789762985.723558:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762985.723560:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762985.723568:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762985.723574:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507726720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762985.723580:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507726848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762985.723590:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762985.723593:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762985.723595:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762985.723598:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2963 00000100:00080000:3.0:1789762985.723601:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762985.723603:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762985.723607:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762985.723617:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507726976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762985.723623:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762985.723627:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762985.723628:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762985.723631:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2963 00000100:00080000:3.0:1789762985.723634:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762985.723636:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789762985.723641:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789762985.723642:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762985.723651:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507727104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762985.723659:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762985.723662:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762985.723663:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762985.723665:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2963 00000100:00080000:3.0:1789762985.723669:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762985.723672:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762985.723675:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762985.723679:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762985/real 1789762985] req@ffff8d25d1b09c00 x1876699507726720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762985.723684:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507727232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762985.723704:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762985.723707:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762985.723710:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789762985.723711:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507726720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762985.723716:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2963 00000100:00080000:3.0:1789762985.723721:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762985.723728:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762985.723732:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762985.723735:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507726720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789762985.723755:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762985.723777:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762985.723791:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762985/real 1789762985] req@ffff8d25d1b0ad80 x1876699507726848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762985.723806:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507726848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762985.723820:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762985.723820:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789762985.723829:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789762985.723830:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762985.723834:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762985.723841:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762985.723846:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507727360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789762985.723866:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762985.723879:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762985/real 1789762985] req@ffff8d25d1b09180 x1876699507726976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762985.723888:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507726976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762985.723898:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762985.723901:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762985.723903:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507726976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762985.723907:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789762985.723919:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762985.723926:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762985/real 1789762985] req@ffff8d25d1b0aa00 x1876699507727104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762985.723948:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507727104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762985.723957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762985.723961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762985.723964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762985.723972:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507727232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762985.723996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762985.724009:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762985/real 1789762985] req@ffff8d25d1b08a80 x1876699507727232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762985.724018:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507727232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762985.724024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762985.724026:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762985.724029:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507727232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762985.724048:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507727360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789762985.724066:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789762985.724080:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762985/real 1789762985] req@ffff8d25d1b0bb80 x1876699507727360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762985.724090:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507727360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763010 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789762985.724096:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789762985.724098:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789762985.724100:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507727360:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789762985.787473:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762986.811490:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762987.835513:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762988.859542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762989.883505:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789762990.843593:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762990.843606:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789762990.843613:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762990.843629:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2968 00000100:00080000:2.0:1789762990.843644:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762990.843666:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762990.843736:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507727488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762990.843791:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762990.843796:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762990.843800:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762990.843806:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2968 00000100:00080000:2.0:1789762990.843813:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762990.843821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762990.843834:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762990.843853:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507727616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762990.843873:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762990.843887:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789762990.843891:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507727488:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789762990.843894:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762990.843913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2968 00000100:00080000:2.0:1789762990.843933:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762990.843939:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762990.843944:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762990.843960:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507727744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762990.843972:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762990.843977:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789762990.843979:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762990.843984:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2968 00000100:00080000:2.0:1789762990.843997:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762990.844001:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762990.844006:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789762990.844020:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507727872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762990.844032:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762990.844037:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789762990.844039:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762990.844044:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2968 00000100:00080000:2.0:1789762990.844049:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762990.844057:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789762990.844068:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789762990.844073:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762990.844092:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507728000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789762990.844110:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789762990.844115:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789762990.844118:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789762990.844124:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2968 00000100:00080000:2.0:1789762990.844129:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789762990.844135:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789762990.844147:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762990.844159:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762990/real 1789762990] req@ffff8d25c8020700 x1876699507727488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789762990.844168:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507728128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789762990.844312:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507727488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762990.844332:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762990.844354:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762990.844358:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507727488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762990.844387:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762990.844499:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762990.844517:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762990/real 1789762990] req@ffff8d25c8020e00 x1876699507727616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762990.844530:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507727616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762990.844537:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762990.844539:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762990.844543:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762990.844569:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762990.844601:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762990.844616:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762990/real 1789762990] req@ffff8d25c8021500 x1876699507727744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762990.844632:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507727744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762990.844642:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762990.844646:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762990.844665:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762990.844670:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789762990.844685:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762990.844692:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762990/real 1789762990] req@ffff8d25c8021c00 x1876699507727872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762990.844700:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507727872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762990.844706:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762990.844708:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762990.844710:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507727872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762990.844732:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762990.844754:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762990.844768:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762990/real 1789762990] req@ffff8d25c8021880 x1876699507728000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762990.844787:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507728000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762990.844805:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762990.844811:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762990.844819:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762990.844831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789762990.844873:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789762990.844897:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762990/real 1789762990] req@ffff8d25c8023480 x1876699507728128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762990.844913:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507728128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763015 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762990.844924:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762990.844953:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762990.844955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728128:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762990.907553:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762991.931584:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762992.955565:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762993.979534:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762995.003545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789762995.963546:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762995.963561:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789762995.963567:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762995.963580:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2973 00000100:00080000:3.0:1789762995.963593:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762995.963604:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789762995.963649:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507728256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762995.963678:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762995.963681:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762995.963682:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762995.963686:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2973 00000100:00080000:3.0:1789762995.963689:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762995.963694:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762995.963705:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762995.963715:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507728384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762995.963726:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507728256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789762995.963730:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762995.963742:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762995.963745:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762995.963749:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2973 00000100:00080000:3.0:1789762995.963751:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762995.963755:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762995.963758:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762995.963767:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507728512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762995.963774:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762995.963777:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789762995.963779:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762995.963783:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2973 00000100:00080000:3.0:1789762995.963799:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762995.963803:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762995.963809:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789762995.963817:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507728640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762995.963825:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762995.963828:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789762995.963830:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762995.963836:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2973 00000100:00080000:3.0:1789762995.963841:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762995.963844:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789762995.963848:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789762995.963853:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507728384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762995.963860:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507728768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789762995.963871:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789762995.963874:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789762995.963876:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789762995.963883:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2973 00000100:00080000:3.0:1789762995.963888:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789762995.963891:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789762995.963896:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507728512:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789762995.963899:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789762995.963918:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507728896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789762995.963942:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507728640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789762995.963973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789762995.963990:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728896:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762996.027577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789762997.051546:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789762998.075600:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:2.0:1789762999.035735:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:2.0:1789762999.035796:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:2.0:1789762999.035815:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762999.035856:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762999.035868:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762999.035877:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762999.035886:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:2.0:1789762999.035894:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789762999.035907:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762995/real 1789762999] req@ffff8d25d1b08700 x1876699507728384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762999.035949:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507728384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762999.035964:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762999.035971:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762999.035976:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507728384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762999.035986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762995/real 1789762999] req@ffff8d25d1b0a680 x1876699507728512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762999.036000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507728512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762999.036009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762999.036014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762999.036018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507728512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762999.036025:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762995/real 1789762999] req@ffff8d25d1b0b800 x1876699507728640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762999.036041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507728640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762999.036050:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762999.036056:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762999.036059:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507728640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789762999.036066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762995/real 1789762999] req@ffff8d25d1b09880 x1876699507728256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762999.036080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507728256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762999.036089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762999.036093:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762999.036097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507728256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789762999.036156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762995/real 1789762999] req@ffff8d25d1b08e00 x1876699507728768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762999.036171:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507728768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762999.036180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762999.036184:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762999.036187:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789762999.036194:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789762995/real 1789762999] req@ffff8d25d1b0bb80 x1876699507728896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789762999.036208:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507728896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763020 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789762999.036216:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789762999.036221:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789762999.036225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507728896:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789762999.099535:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763000.123583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763001.083469:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763001.083480:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763001.083484:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763001.083496:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2978 00000100:00080000:3.0:1789763001.083507:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763001.083515:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763001.083553:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507729024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763001.083589:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763001.083598:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763001.083601:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763001.083606:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2978 00000100:00080000:3.0:1789763001.083612:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763001.083617:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763001.083629:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763001.083646:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507729152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763001.083659:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507729024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763001.083664:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763001.083674:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763001.083676:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763001.083683:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2978 00000100:00080000:3.0:1789763001.083689:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763001.083695:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763001.083701:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763001.083720:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507729280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763001.083741:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763001.083749:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763001.083751:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763001.083756:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763001.083763:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2978 00000100:00080000:3.0:1789763001.083769:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763001.083775:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763001.083782:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763001.083800:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507729408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763001.083812:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763001.083821:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763001.083822:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffff8d25d1b0bb80 x1876699507729024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763001.083824:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763001.083836:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2978 00000100:00080000:3.0:1789763001.083847:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763001.083855:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763001.083863:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763001.083864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507729024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.083885:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763001.083886:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507729536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763001.083897:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763001.083910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507729024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763001.083933:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763001.083938:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763001.083940:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763001.083946:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2978 00000100:00080000:3.0:1789763001.083952:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763001.083957:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729152:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763001.083964:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763001.083973:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763001.083991:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507729664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763001.084005:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763001.084033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffff8d25d1b08e00 x1876699507729152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.084048:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507729152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.084057:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.084061:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763001.084064:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763001.084072:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763001.084121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763001.084134:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffff8d25d1b09880 x1876699507729280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.084145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507729280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.084153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.084157:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763001.084161:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763001.084169:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763001.084202:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763001.084215:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffff8d25d1b0b800 x1876699507729408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.084227:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507729408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.084236:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.084240:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763001.084243:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763001.084249:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507729536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763001.084270:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763001.084282:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffff8d25d1b0a680 x1876699507729536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.084294:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507729536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.084303:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.084307:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763001.084310:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507729536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763001.084335:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507729664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763001.084357:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763001.084370:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763001/real 1789763001] req@ffff8d25d1b08700 x1876699507729664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763001.084418:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507729664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763026 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763001.084423:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763001.084425:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763001.084428:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507729664:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763001.147536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763002.171553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763003.195523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763004.219629:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763005.243520:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763006.203439:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763006.203457:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763006.203461:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763006.203472:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2983 00000100:00080000:3.0:1789763006.203481:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763006.203489:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763006.203528:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507729792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763006.203552:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763006.203555:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763006.203556:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763006.203560:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2983 00000100:00080000:3.0:1789763006.203563:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763006.203567:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763006.203576:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763006.203586:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507729920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763006.203595:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763006.203597:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507729792:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763006.203600:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763006.203605:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763006.203609:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2983 00000100:00080000:3.0:1789763006.203611:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763006.203614:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763006.203617:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763006.203623:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507730048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763006.203635:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763006.203638:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763006.203639:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763006.203642:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2983 00000100:00080000:3.0:1789763006.203645:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763006.203648:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763006.203651:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763006.203658:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507730176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763006.203671:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763006.203674:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763006.203675:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763006.203678:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2983 00000100:00080000:3.0:1789763006.203681:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763006.203683:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763006.203686:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763006.203693:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507730304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763006.203700:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763006.203709:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763006.203713:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763006.203714:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763006.203718:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2983 00000100:00080000:3.0:1789763006.203721:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763006.203723:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763006.203726:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763006.203740:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507730432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763006.203757:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffff8d25d1b08a80 x1876699507729792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763006.203776:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507729792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763006.203785:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763006.203788:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763006.203793:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507729792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763006.203821:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763006.203846:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763006.203857:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffff8d25d1b0aa00 x1876699507729920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763006.203871:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507729920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763006.203877:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763006.203879:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763006.203882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507729920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763006.203885:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763006.203897:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763006.203904:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffff8d25d1b09180 x1876699507730048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763006.203912:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507730048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763006.203917:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763006.203919:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763006.203921:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763006.203924:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763006.203935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763006.203941:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffff8d25d1b0ad80 x1876699507730176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763006.203948:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507730176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763006.203958:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763006.203961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763006.203963:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763006.203966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507730304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763006.203976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763006.203982:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffff8d25d1b09c00 x1876699507730304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763006.203989:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507730304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763006.203995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763006.203997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763006.203999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507730304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763006.204002:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507730432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763006.204012:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763006.204018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763006/real 1789763006] req@ffff8d25d1b0b480 x1876699507730432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763006.204026:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507730432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763031 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763006.204031:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763006.204032:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763006.204035:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507730432:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763006.267511:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763007.291520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763008.315511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763009.339527:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763010.363520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763011.323483:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763011.323502:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763011.323506:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763011.323514:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2989 00000100:00080000:2.0:1789763011.323522:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763011.323527:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763011.323559:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507730560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763011.323584:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763011.323587:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763011.323588:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763011.323592:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2989 00000100:00080000:2.0:1789763011.323594:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763011.323597:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763011.323605:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763011.323614:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507730688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763011.323622:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763011.323626:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763011.323629:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763011.323634:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2989 00000100:00100000:3.0:1789763011.323634:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507730560:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763011.323643:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763011.323653:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763011.323657:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763011.323671:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507730816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763011.323679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763011.323682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763011.323682:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763011.323685:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2989 00000100:00080000:2.0:1789763011.323687:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763011.323689:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763011.323694:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763011.323701:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507730944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763011.323709:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763011.323711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763011.323712:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763011.323714:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2989 00000100:00080000:2.0:1789763011.323718:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763011.323721:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763011.323724:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763011.323731:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507731072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763011.323739:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763011.323751:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763011.323755:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763011.323757:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763011.323761:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2989 00000100:00080000:2.0:1789763011.323766:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763011.323769:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763011.323773:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763011.323780:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507731200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763011.323797:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffff8d25c8023100 x1876699507730560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.323815:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507730560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.323824:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.323828:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763011.323831:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507730560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763011.323855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763011.323873:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763011.323881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffff8d25c8022d80 x1876699507730688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.323890:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507730688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.323896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.323898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763011.323901:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763011.323904:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763011.323915:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763011.323921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffff8d25c8020380 x1876699507730816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.323929:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507730816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.323939:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.323941:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763011.323944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763011.323947:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763011.323958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763011.323963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffff8d25c8022300 x1876699507730944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.323970:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507730944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.323976:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.323978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763011.323980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507730944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763011.323984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763011.323994:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763011.324000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffff8d25c8023b80 x1876699507731072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.324007:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507731072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.324013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.324014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763011.324017:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763011.324037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763011.324054:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763011.324062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763011/real 1789763011] req@ffff8d25c8023800 x1876699507731200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763011.324070:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507731200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763036 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763011.324076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763011.324078:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763011.324080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731200:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763011.387514:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763012.411545:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763013.435496:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763014.459611:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763015.483575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763016.443527:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763016.443542:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763016.443548:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763016.443563:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2994 00000100:00080000:3.0:1789763016.443579:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763016.443590:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763016.443653:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507731328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763016.443712:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763016.443720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763016.443724:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763016.443735:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2994 00000100:00080000:3.0:1789763016.443746:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763016.443754:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763016.443766:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507731328:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789763016.443769:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763016.443801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507731456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763016.443819:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763016.443824:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763016.443827:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763016.443833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2994 00000100:00080000:3.0:1789763016.443840:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763016.443845:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789763016.443855:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763016.443857:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763016.443878:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507731584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763016.443894:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763016.443899:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:2.0:1789763016.443909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763016/real 1789763016] req@ffff8d25d1b0a300 x1876699507731328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763016.443918:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763016.443926:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2994 00000100:00080000:3.0:1789763016.443931:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763016.443933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507731328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763016.443944:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763016.443964:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1789763016.443968:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763016.443978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763016.444004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507731328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763016.444007:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507731712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763016.444037:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763016.444041:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763016.444049:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763016.444051:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507731456:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763016.444059:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2994 00000100:00080000:3.0:1789763016.444066:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763016.444073:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763016.444082:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763016.444093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763016.444103:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507731840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763016.444116:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763016/real 1789763016] req@ffff8d25d1b0b100 x1876699507731456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763016.444136:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763016.444145:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763016.444151:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763016.444153:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507731456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763016.444165:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2994 00000100:00080000:3.0:1789763016.444180:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763016.444184:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763016.444193:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763016.444201:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789763016.444208:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763016.444209:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507731456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763016.444232:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507731584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763016.444239:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507731968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763016.444254:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763016.444272:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763016/real 1789763016] req@ffff8d25d1b08380 x1876699507731584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763016.444284:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507731584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763016.444291:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763016.444295:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763016.444301:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507731584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763016.444306:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507731712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763016.444318:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763016.444325:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763016/real 1789763016] req@ffff8d25d1b09500 x1876699507731712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763016.444339:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507731712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763016.444345:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763016.444347:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763016.444349:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507731712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763016.444365:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763016.444401:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763016.444411:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763016/real 1789763016] req@ffff8d25d1b08000 x1876699507731840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763016.444420:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507731840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763016.444426:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763016.444428:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763016.444431:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763016.444435:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763016.444448:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763016.444455:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763016/real 1789763016] req@ffff8d25d1b0a300 x1876699507731968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763016.444463:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507731968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763041 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763016.444474:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763016.444477:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763016.444479:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507731968:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763016.507604:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763017.531604:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763018.555552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763019.579648:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763020.603659:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763021.563473:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763021.563486:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763021.563493:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763021.563507:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 2999 00000100:00080000:3.0:1789763021.563520:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763021.563557:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763021.563623:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507732096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763021.563657:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763021.563662:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763021.563664:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763021.563676:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2999 00000100:00080000:3.0:1789763021.563680:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763021.563686:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763021.563701:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763021.563716:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507732224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763021.563727:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763021.563732:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763021.563735:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763021.563737:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507732096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763021.563742:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2999 00000100:00080000:3.0:1789763021.563749:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763021.563754:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763021.563758:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763021.563769:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507732352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763021.563785:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763021.563789:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763021.563791:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763021.563794:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 2999 00000100:00080000:3.0:1789763021.563798:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763021.563800:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763021.563805:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763021.563814:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507732480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763021.563825:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763021.563829:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763021.563830:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763021.563833:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2999 00000100:00080000:3.0:1789763021.563837:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763021.563843:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763021.563846:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763021.563852:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763021.563863:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507732608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763021.563873:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763021.563877:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763021.563879:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763021.563882:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 2999 00000100:00080000:3.0:1789763021.563886:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763021.563890:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763021.563894:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763021.563907:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507732736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763021.563920:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763021/real 1789763021] req@ffff8d25d1b0b480 x1876699507732096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763021.563987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507732096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763021.564004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763021.564011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763021.564017:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507732096:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763021.564054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763021.564085:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763021.564098:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763021/real 1789763021] req@ffff8d25d1b09c00 x1876699507732224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763021.564109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507732224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763021.564124:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763021.564129:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763021.564132:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763021.564139:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763021.564159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763021.564173:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763021/real 1789763021] req@ffff8d25d1b0ad80 x1876699507732352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763021.564192:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507732352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763021.564204:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763021.564209:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763021.564215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763021.564225:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732480:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763021.564259:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763021.564277:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763021/real 1789763021] req@ffff8d25d1b09180 x1876699507732480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763021.564293:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507732480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763021.564306:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763021.564311:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763021.564318:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763021.564328:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507732608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763021.564357:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763021.564374:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763021/real 1789763021] req@ffff8d25d1b0aa00 x1876699507732608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763021.564455:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507732608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763021.564466:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763021.564473:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763021.564478:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507732608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763021.564487:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507732736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763021.564515:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763021.564527:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763021/real 1789763021] req@ffff8d25d1b08a80 x1876699507732736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763021.564540:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507732736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763046 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763021.564547:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763021.564552:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763021.564556:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507732736:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763021.627569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763022.651661:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763023.675623:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763024.699577:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763025.723625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763026.683592:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763026.683606:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763026.683612:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763026.683625:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3004 00000100:00080000:2.0:1789763026.683639:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763026.683649:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763026.683698:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507732864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763026.683745:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763026.683778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763026.683782:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763026.683789:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3004 00000100:00080000:2.0:1789763026.683794:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763026.683801:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763026.683814:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763026.683835:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507732992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763026.683843:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507732864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763026.683854:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763026.683876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763026.683880:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763026.683886:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3004 00000100:00080000:2.0:1789763026.683890:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763026.683894:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763026.683902:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763026.683944:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507733120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763026.683958:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763026.683963:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763026.683967:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763026.683972:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3004 00000100:00080000:2.0:1789763026.684005:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763026.684012:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763026.684021:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763026.684039:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507733248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763026.684046:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763026.684070:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763026.684075:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763026.684078:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763026.684085:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3004 00000100:00080000:2.0:1789763026.684092:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763026.684097:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763026.684103:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763026.684123:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507733376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763026.684143:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763026.684146:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763026/real 1789763026] req@ffff8d25c8022a00 x1876699507732864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763026.684164:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763026.684180:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763026.684200:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3004 00000100:00100000:3.0:1789763026.684201:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507732864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763026.684224:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763026.684238:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763026.684239:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789763026.684255:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789763026.684258:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763026.684268:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507732864:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763026.684289:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507733504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763026.684317:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763026.684360:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763026.684414:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763026/real 1789763026] req@ffff8d25c8020000 x1876699507732992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763026.684432:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507732992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763026.684453:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763026.684458:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763026.684463:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507732992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763026.684471:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763026.684497:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763026.684511:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763026/real 1789763026] req@ffff8d25c8020a80 x1876699507733120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763026.684525:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507733120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763026.684535:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763026.684539:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763026.684542:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763026.684550:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763026.684572:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763026.684584:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763026/real 1789763026] req@ffff8d25c8021180 x1876699507733248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763026.684598:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507733248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763026.684608:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763026.684611:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763026.684616:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763026.684624:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507733376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763026.684644:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763026.684656:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763026/real 1789763026] req@ffff8d25c8022680 x1876699507733376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763026.684670:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507733376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763026.684680:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763026.684683:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763026.684687:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507733376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763026.684694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507733504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763026.684713:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763026.684725:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763026/real 1789763026] req@ffff8d25c8023800 x1876699507733504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763026.684738:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507733504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763051 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763026.684748:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763026.684752:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763026.684755:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507733504:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763026.747629:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763027.771629:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763028.795626:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763029.819592:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763030.843674:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763031.803606:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763031.803622:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763031.803629:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763031.803646:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3009 00000100:00080000:3.0:1789763031.803662:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763031.803676:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763031.803753:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507733632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763031.803812:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763031.803821:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763031.803825:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763031.803834:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3009 00000100:00080000:3.0:1789763031.803842:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763031.803853:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763031.803892:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763031.803914:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507733632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763031.803924:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507733760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763031.803944:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763031.803949:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763031.803952:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763031.803958:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3009 00000100:00080000:3.0:1789763031.803963:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763031.803969:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763031.803976:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763031.803991:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507733888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763031.804004:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763031.804010:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763031.804013:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763031.804017:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3009 00000100:00080000:3.0:1789763031.804032:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763031.804037:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763031.804043:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763031.804059:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763031.804062:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507734016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763031.804080:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763031.804085:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763031.804088:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763031.804094:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3009 00000100:00080000:3.0:1789763031.804099:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763031.804104:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763031.804110:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763031.804124:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507734144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763031.804137:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763031.804144:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763031/real 1789763031] req@ffff8d25d1b08700 x1876699507733632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763031.804146:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763031.804152:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763031.804166:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3009 00000100:00080000:3.0:1789763031.804177:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763031.804187:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763031.804194:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763031.804202:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507733632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763031.804213:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507734272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763031.804231:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763031.804242:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763031.804251:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507733632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763031.804293:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763031.804325:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763031.804342:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763031/real 1789763031] req@ffff8d25d1b0a680 x1876699507733760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763031.804368:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507733760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763031.804412:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763031.804416:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763031.804421:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763031.804428:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763031.804457:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763031.804473:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763031/real 1789763031] req@ffff8d25d1b0b800 x1876699507733888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763031.804488:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507733888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763031.804499:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763031.804503:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763031.804507:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507733888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763031.804514:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763031.804534:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763031.804546:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763031/real 1789763031] req@ffff8d25d1b09880 x1876699507734016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763031.804560:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507734016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763031.804579:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763031.804583:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763031.804587:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763031.804594:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507734144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763031.804621:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763031.804642:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763031/real 1789763031] req@ffff8d25d1b08e00 x1876699507734144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763031.804667:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507734144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763031.804684:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763031.804690:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763031.804698:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507734144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763031.804710:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507734272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763031.804748:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763031.804764:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763031/real 1789763031] req@ffff8d25d1b0bb80 x1876699507734272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763031.804780:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507734272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763056 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763031.804796:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763031.804803:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763031.804810:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507734272:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763031.867649:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763032.891722:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763033.915707:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763034.939628:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763035.963470:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763036.923452:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763036.923463:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763036.923468:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763036.923480:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3014 00000100:00080000:3.0:1789763036.923490:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763036.923498:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763036.923540:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507734400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763036.923570:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763036.923573:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763036.923574:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763036.923578:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3014 00000100:00080000:3.0:1789763036.923581:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763036.923584:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763036.923595:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763036.923604:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507734528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763036.923618:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763036.923622:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763036.923625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507734400:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763036.923625:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763036.923632:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3014 00000100:00080000:3.0:1789763036.923635:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763036.923638:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763036.923640:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763036.923648:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507734656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763036.923655:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763036.923658:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763036.923659:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763036.923661:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3014 00000100:00080000:3.0:1789763036.923663:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763036.923666:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763036.923669:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763036.923676:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507734784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763036.923683:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763036.923686:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763036.923687:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763036.923690:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3014 00000100:00080000:3.0:1789763036.923693:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763036.923696:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763036.923699:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763036.923704:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763036.923709:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507734912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763036.923717:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763036.923720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763036.923722:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763036.923725:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3014 00000100:00080000:3.0:1789763036.923727:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763036.923729:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763036.923732:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763036.923738:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507735040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763036.923760:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763036/real 1789763036] req@ffff8d25d1b0bb80 x1876699507734400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763036.923777:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507734400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763036.923786:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763036.923790:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763036.923793:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507734400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763036.923815:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763036.923850:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763036.923862:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763036/real 1789763036] req@ffff8d25d1b08e00 x1876699507734528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763036.923871:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507734528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763036.923877:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763036.923879:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763036.923882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763036.923888:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763036.923903:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763036.923911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763036/real 1789763036] req@ffff8d25d1b09880 x1876699507734656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763036.923925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507734656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763036.923931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763036.923933:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763036.923935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763036.923939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763036.923951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763036.923957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763036/real 1789763036] req@ffff8d25d1b0b800 x1876699507734784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763036.923965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507734784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763036.923970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763036.923972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763036.923974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507734784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763036.923978:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507734912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763036.923989:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763036.923995:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763036/real 1789763036] req@ffff8d25d1b0a680 x1876699507734912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763036.924003:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507734912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763036.924008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763036.924010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763036.924012:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507734912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763036.924015:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507735040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763036.924026:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763036.924031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763036/real 1789763036] req@ffff8d25d1b08700 x1876699507735040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763036.924039:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507735040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763061 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763036.924044:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763036.924046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763036.924047:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507735040:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763036.987524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763038.011651:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763039.035666:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763040.059657:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763041.083660:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763042.043515:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763042.043529:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763042.043534:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763042.043545:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3019 00000100:00080000:3.0:1789763042.043575:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763042.043586:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763042.043662:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507735168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763042.043711:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763042.043716:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763042.043718:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763042.043724:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3019 00000100:00080000:3.0:1789763042.043728:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763042.043733:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763042.043746:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763042.043760:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507735296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763042.043773:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763042.043778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763042.043781:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763042.043786:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3019 00000100:00080000:3.0:1789763042.043789:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763042.043824:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763042.043833:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763042.043837:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507735168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763042.043871:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507735424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763042.043880:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763042.043882:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763042.043884:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763042.043887:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3019 00000100:00080000:3.0:1789763042.043891:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763042.043894:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763042.043899:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763042.043907:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507735552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763042.043914:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763042.043917:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763042.043918:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763042.043921:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3019 00000100:00080000:3.0:1789763042.043923:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763042.043925:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763042.043928:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763042.043935:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507735680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763042.043942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763042.043946:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763042.043950:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763042.043952:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763042.043956:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3019 00000100:00080000:3.0:1789763042.043959:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763042.043961:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763042.043965:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763042.043972:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507735808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763042.044011:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffff8d25d1b08a80 x1876699507735168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763042.044035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507735168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763042.044044:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763042.044047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763042.044051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507735168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763042.044057:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507735296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763042.044082:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763042.044092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffff8d25d1b0aa00 x1876699507735296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763042.044101:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507735296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763042.044108:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763042.044110:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763042.044112:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507735296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763042.044139:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507735424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763042.044152:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763042.044159:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffff8d25d1b09180 x1876699507735424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763042.044168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507735424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763042.044179:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763042.044181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763042.044183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507735424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763042.044187:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507735552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763042.044198:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763042.044225:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffff8d25d1b0ad80 x1876699507735552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763042.044237:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507735552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763042.044246:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763042.044249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763042.044253:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507735552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763042.044259:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507735680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763042.044277:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763042.044287:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffff8d25d1b09c00 x1876699507735680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763042.044299:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507735680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763042.044307:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763042.044310:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763042.044314:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507735680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763042.044321:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507735808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763042.044338:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763042.044347:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763042/real 1789763042] req@ffff8d25d1b0b480 x1876699507735808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763042.044359:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507735808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763067 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763042.044368:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763042.044370:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763042.044373:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507735808:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763042.107552:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763043.131670:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763044.155597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763045.179495:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763046.203509:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763047.163514:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763047.163531:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763047.163538:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763047.163554:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3024 00000100:00080000:3.0:1789763047.163570:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763047.163582:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763047.163629:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507735936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763047.163667:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763047.163672:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763047.163675:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763047.163683:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3024 00000100:00080000:3.0:1789763047.163690:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763047.163697:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763047.163713:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763047.163737:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507736064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763047.163787:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763047.163795:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763047.163799:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763047.163809:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3024 00000100:00080000:3.0:1789763047.163817:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763047.163830:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763047.163839:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763047.163858:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507736192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763047.163874:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763047.163879:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763047.163882:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763047.163894:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3024 00000100:00080000:3.0:1789763047.163901:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763047.163905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507735936:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763047.163908:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763047.163921:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763047.163936:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507736320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763047.163950:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763047.163954:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763047.163958:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763047.163963:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3024 00000100:00080000:3.0:1789763047.163968:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763047.163972:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763047.163977:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763047.163996:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507736448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763047.164008:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763047.164013:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763047.164015:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763047.164019:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3024 00000100:00080000:3.0:1789763047.164024:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763047.164030:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763047.164039:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763047.164051:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763047.164059:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507736576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763047.164160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffff8d25d1b0a300 x1876699507735936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763047.164197:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507735936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763047.164221:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763047.164230:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763047.164238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507735936:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763047.164257:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763047.164307:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763047.164329:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffff8d25d1b08000 x1876699507736064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763047.164346:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507736064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763047.164358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763047.164363:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763047.164367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763047.164480:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763047.164510:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763047.164526:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffff8d25d1b09500 x1876699507736192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763047.164541:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507736192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763047.164552:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763047.164557:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763047.164561:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763047.164568:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763047.164588:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763047.164601:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffff8d25d1b08380 x1876699507736320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763047.164616:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507736320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763047.164626:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763047.164630:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763047.164633:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763047.164641:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507736448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763047.164660:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763047.164672:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffff8d25d1b0b100 x1876699507736448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763047.164687:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507736448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763047.164697:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763047.164700:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763047.164704:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507736448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763047.164710:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507736576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763047.164729:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763047.164741:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763047/real 1789763047] req@ffff8d25d1b0b480 x1876699507736576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763047.164786:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507736576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763072 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763047.164818:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763047.164826:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763047.164834:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507736576:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763047.227605:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763048.251663:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763049.275568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763050.299562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763051.323575:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763052.283510:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763052.283527:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763052.283535:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763052.283553:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3029 00000100:00080000:3.0:1789763052.283568:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763052.283579:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763052.283642:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507736704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763052.283689:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763052.283695:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763052.283698:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763052.283705:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3029 00000100:00080000:3.0:1789763052.283710:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763052.283716:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763052.283753:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763052.283773:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507736832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763052.283796:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763052.283802:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763052.283804:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763052.283812:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3029 00000100:00080000:3.0:1789763052.283820:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763052.283834:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763052.283836:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507736704:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789763052.283853:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763052.283872:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507736960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763052.283889:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763052.283894:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763052.283897:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763052.283904:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3029 00000100:00080000:3.0:1789763052.283926:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763052.283933:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763052.283942:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763052.283951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763052.283969:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507737088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763052.283987:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763052.283992:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763052.283996:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763052.284006:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3029 00000100:00080000:3.0:1789763052.284017:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763052.284025:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763052.284033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffff8d25d1b09c00 x1876699507736704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763052.284038:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763052.284068:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507737216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763052.284083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507736704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763052.284102:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763052.284132:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763052.284140:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763052.284160:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3029 00000100:00080000:2.0:1789763052.284162:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763052.284166:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763052.284178:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763052.284186:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763052.284205:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763052.284216:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507737344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763052.284230:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507736704:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763052.284264:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763052.284309:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763052.284331:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffff8d25d1b0ad80 x1876699507736832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763052.284348:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507736832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763052.284369:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763052.284373:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763052.284441:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763052.284479:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763052.284507:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763052.284523:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffff8d25d1b09180 x1876699507736960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763052.284538:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507736960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763052.284548:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763052.284553:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763052.284557:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507736960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763052.284564:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763052.284583:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763052.284595:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffff8d25d1b0aa00 x1876699507737088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763052.284609:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507737088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763052.284619:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763052.284623:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763052.284643:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763052.284652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507737216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763052.284675:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763052.284689:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffff8d25d1b08a80 x1876699507737216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763052.284703:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507737216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763052.284714:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763052.284741:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763052.284749:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507737216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763052.284763:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507737344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763052.284805:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763052.284829:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763052/real 1789763052] req@ffff8d25d1b08700 x1876699507737344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763052.284851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507737344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763077 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763052.284862:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763052.284866:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763052.284870:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507737344:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763052.347621:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763053.371621:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763054.395611:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763055.419631:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763056.443586:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763057.403556:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763057.403573:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763057.403580:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763057.403596:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3035 00000100:00080000:3.0:1789763057.403610:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763057.403621:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763057.403701:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507737472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763057.403750:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763057.403755:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763057.403758:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763057.403765:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3035 00000100:00080000:3.0:1789763057.403770:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763057.403777:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763057.403792:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763057.403810:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507737600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763057.403825:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763057.403830:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763057.403835:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763057.403842:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3035 00000100:00080000:3.0:1789763057.403847:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763057.403854:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763057.403861:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763057.403886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507737472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763057.403887:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507737728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763057.403926:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763057.403932:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763057.403935:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763057.403944:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3035 00000100:00080000:3.0:1789763057.403949:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763057.403954:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763057.403962:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763057.403978:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507737856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763057.403995:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763057.403997:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763057.404010:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763057.404014:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763057.404023:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3035 00000100:00080000:3.0:1789763057.404030:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763057.404037:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763057.404043:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763057.404060:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507737984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763057.404066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763057/real 1789763057] req@ffff8d25d1b0a680 x1876699507737472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763057.404122:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763057.404129:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763057.404132:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763057.404142:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507737472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763057.404146:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3035 00000100:00080000:3.0:1789763057.404160:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763057.404179:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763057.404183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763057.404187:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763057.404192:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763057.404204:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507737472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763057.404224:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763057.404225:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507738112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763057.404270:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763057.404287:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763057/real 1789763057] req@ffff8d25d1b0b800 x1876699507737600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763057.404302:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507737600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763057.404309:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763057.404313:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763057.404316:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763057.404348:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763057.404367:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763057.404431:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763057/real 1789763057] req@ffff8d25d1b09880 x1876699507737728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763057.404451:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507737728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763057.404460:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763057.404463:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763057.404472:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763057.404481:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763057.404511:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763057.404522:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763057/real 1789763057] req@ffff8d25d1b08e00 x1876699507737856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763057.404534:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507737856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763057.404541:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763057.404544:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763057.404547:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507737856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763057.404554:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507737984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763057.404579:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763057.404594:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763057/real 1789763057] req@ffff8d25d1b0bb80 x1876699507737984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763057.404609:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507737984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763057.404625:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763057.404629:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763057.404633:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507737984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763057.404651:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507738112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763057.404713:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763057.404740:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763057/real 1789763057] req@ffff8d25d1b08700 x1876699507738112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763057.404753:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507738112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763082 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763057.404761:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763057.404770:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763057.404774:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507738112:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763057.467703:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763058.491725:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763059.515604:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763060.539642:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763061.563607:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763062.523499:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763062.523521:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763062.523530:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763062.523547:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3040 00000100:00080000:3.0:1789763062.523562:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763062.523573:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763062.523632:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507738240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763062.523694:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763062.523700:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763062.523702:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763062.523709:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3040 00000100:00080000:3.0:1789763062.523715:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763062.523721:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763062.523737:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763062.523757:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507738368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763062.523784:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763062.523789:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763062.523796:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763062.523798:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507738240:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763062.523806:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3040 00000100:00080000:3.0:1789763062.523816:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763062.523823:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763062.523830:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763062.523847:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507738496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763062.523860:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763062.523866:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763062.523868:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763062.523873:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3040 00000100:00080000:3.0:1789763062.523896:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763062.523902:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763062.523908:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763062.523937:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507738624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763062.523972:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763062.523980:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763062.523988:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507738368:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763062.523994:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763062.524007:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3040 00000100:00080000:3.0:1789763062.524017:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763062.524028:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763062.524038:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507738496:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789763062.524040:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763062.524077:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507738752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763062.524083:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507738624:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789763062.524121:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763062.524127:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507738752:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763062.524135:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763062.524143:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763062.524154:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3040 00000100:00080000:3.0:1789763062.524165:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763062.524173:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763062.524183:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763062.524208:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507738880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763062.524307:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507738880:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763062.587635:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763063.611720:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763064.635662:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763065.595836:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1789763065.595894:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763065.595919:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763065.595998:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763065.596030:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763065.596051:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763065.596070:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763065.596091:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789763065.596113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763062/real 1789763065] req@ffff8d25d1b09c00 x1876699507738752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763065.596162:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507738752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763065.596184:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763065.596192:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763065.596201:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507738752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763065.596216:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763062/real 1789763065] req@ffff8d25d1b0aa00 x1876699507738368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763065.596243:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507738368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763065.596261:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763065.596269:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763065.596276:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507738368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763065.596290:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763062/real 1789763065] req@ffff8d25d1b09180 x1876699507738496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763065.596315:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507738496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763065.596332:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763065.596340:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763065.596346:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507738496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763065.596358:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763062/real 1789763065] req@ffff8d25d1b0ad80 x1876699507738624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763065.596434:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507738624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763065.596450:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763065.596456:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763065.596462:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507738624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763065.596475:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763062/real 1789763065] req@ffff8d25d1b08a80 x1876699507738240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763065.596498:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507738240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763065.596530:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763065.596536:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763065.596543:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507738240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763065.596643:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763062/real 1789763065] req@ffff8d25d1b0b480 x1876699507738880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763065.596679:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507738880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763087 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763065.596697:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763065.596706:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763065.596713:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507738880:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763065.659570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763066.683685:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763067.643511:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763067.643527:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763067.643533:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763067.643550:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3045 00000100:00080000:3.0:1789763067.643567:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763067.643579:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763067.643675:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507739008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763067.643730:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763067.643738:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763067.643742:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763067.643751:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3045 00000100:00080000:3.0:1789763067.643759:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763067.643769:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763067.643788:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763067.643823:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507739008:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763067.643824:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507739136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763067.643855:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763067.643860:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763067.643862:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763067.643869:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3045 00000100:00080000:3.0:1789763067.643875:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763067.643880:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763067.643887:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763067.643905:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507739264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763067.643919:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763067.643923:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763067.643925:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763067.643929:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3045 00000100:00080000:3.0:1789763067.643938:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763067.643945:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763067.643954:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763067.643971:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763067.643979:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507739392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763067.644004:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763067.644012:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763067.644016:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763067.644023:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3045 00000100:00080000:3.0:1789763067.644031:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763067.644040:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763067.644050:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763067.644062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763067/real 1789763067] req@ffff8d25d1b0b100 x1876699507739008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763067.644078:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507739520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763067.644106:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763067.644116:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507739008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763067.644117:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763067.644126:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763067.644142:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3045 00000100:00080000:3.0:1789763067.644157:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763067.644158:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763067.644169:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763067.644184:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763067.644187:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763067.644199:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507739008:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763067.644216:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507739648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763067.644268:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763067.644321:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763067.644348:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763067/real 1789763067] req@ffff8d25d1b08380 x1876699507739136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763067.644436:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507739136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763067.644457:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763067.644464:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763067.644472:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763067.644485:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763067.644527:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763067.644551:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763067/real 1789763067] req@ffff8d25d1b09500 x1876699507739264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763067.644570:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507739264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763067.644585:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763067.644591:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763067.644597:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763067.644640:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763067.644681:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763067.644705:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763067/real 1789763067] req@ffff8d25d1b08000 x1876699507739392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763067.644730:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507739392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763067.644746:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763067.644751:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763067.644758:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763067.644770:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507739520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763067.644809:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763067.644832:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763067/real 1789763067] req@ffff8d25d1b0a300 x1876699507739520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763067.644854:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507739520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763067.644869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763067.644875:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763067.644882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507739520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763067.644893:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507739648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763067.644930:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763067.644950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763067/real 1789763067] req@ffff8d25d1b0b480 x1876699507739648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763067.644972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507739648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763092 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763067.644988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763067.644994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763067.644999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507739648:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763067.707641:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763068.731587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763069.755646:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763070.779701:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763071.803569:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763072.763505:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763072.763513:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763072.763517:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763072.763525:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3050 00000100:00080000:2.0:1789763072.763535:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763072.763541:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763072.763587:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507739776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763072.763623:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763072.763627:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763072.763629:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763072.763633:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3050 00000100:00080000:2.0:1789763072.763637:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763072.763641:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763072.763649:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763072.763662:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507739904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763072.763668:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507739776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763072.763678:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763072.763682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763072.763684:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763072.763687:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3050 00000100:00080000:2.0:1789763072.763690:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763072.763694:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763072.763698:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763072.763708:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507740032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763072.763716:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763072.763719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763072.763720:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763072.763723:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3050 00000100:00080000:2.0:1789763072.763739:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763072.763742:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763072.763746:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763072.763760:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763072.763761:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507740160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763072.763773:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763072.763777:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763072.763779:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763072.763783:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3050 00000100:00080000:2.0:1789763072.763787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763072.763791:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763072.763794:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763072.763801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507740288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763072.763808:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763072.763811:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763072.763813:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763072.763819:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3050 00000100:00100000:3.0:1789763072.763820:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763072/real 1789763072] req@ffff8d25c8023b80 x1876699507739776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763072.763827:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763072.763835:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763072.763850:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763072.763864:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507740416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763072.763868:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507739776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763072.763904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763072.763911:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763072.763920:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507739776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763072.763950:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763072.763972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763072.763982:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763072/real 1789763072] req@ffff8d25c8022300 x1876699507739904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763072.763991:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507739904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763072.763998:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763072.764000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763072.764003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507739904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763072.764007:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763072.764020:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763072.764027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763072/real 1789763072] req@ffff8d25c8020380 x1876699507740032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763072.764039:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507740032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763072.764048:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763072.764051:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763072.764054:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763072.764062:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763072.764094:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763072.764102:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763072/real 1789763072] req@ffff8d25c8022d80 x1876699507740160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763072.764111:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507740160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763072.764117:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763072.764119:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763072.764121:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763072.764126:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507740288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763072.764138:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763072.764145:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763072/real 1789763072] req@ffff8d25c8023100 x1876699507740288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763072.764158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507740288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763072.764163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763072.764165:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763072.764167:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507740288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763072.764174:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507740416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763072.764195:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763072.764207:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763072/real 1789763072] req@ffff8d25c8023480 x1876699507740416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763072.764220:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507740416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763097 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763072.764227:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763072.764229:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763072.764231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507740416:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763072.827584:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763073.851597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763074.875506:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763075.899649:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763076.923649:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763077.883587:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763077.883606:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763077.883615:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763077.883635:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3055 00000100:00080000:3.0:1789763077.883654:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763077.883665:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763077.883729:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507740544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763077.883778:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763077.883783:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763077.883786:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763077.883792:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3055 00000100:00080000:3.0:1789763077.883798:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763077.883804:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763077.883820:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763077.883837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507740672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763077.883855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507740544:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763077.883861:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763077.883880:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763077.883884:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763077.883892:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3055 00000100:00080000:3.0:1789763077.883898:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763077.883903:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763077.883910:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763077.883926:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507740800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763077.883939:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763077.883943:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763077.883947:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763077.883953:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3055 00000100:00080000:3.0:1789763077.883961:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763077.883966:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763077.883974:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763077.883982:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763077.883992:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507740928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763077.884022:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763077.884027:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763077.884031:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763077.884039:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3055 00000100:00080000:3.0:1789763077.884044:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763077.884049:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763077.884062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763077/real 1789763077] req@ffff8d25d1b08a80 x1876699507740544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763077.884062:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763077.884093:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507741056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763077.884111:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763077.884112:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507740544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763077.884142:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763077.884150:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763077.884172:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3055 00000100:00080000:2.0:1789763077.884175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763077.884181:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763077.884194:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763077.884204:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763077.884216:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507740544:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789763077.884220:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763077.884252:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507741184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763077.884269:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763077.884323:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763077.884349:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763077/real 1789763077] req@ffff8d25d1b0ad80 x1876699507740672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763077.884367:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507740672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763077.884411:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763077.884418:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763077.884422:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763077.884431:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763077.884458:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763077.884473:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763077/real 1789763077] req@ffff8d25d1b09180 x1876699507740800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763077.884489:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507740800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763077.884499:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763077.884503:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763077.884508:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763077.884515:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763077.884563:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763077.884578:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763077/real 1789763077] req@ffff8d25d1b0aa00 x1876699507740928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763077.884593:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507740928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763077.884604:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763077.884608:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763077.884613:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507740928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763077.884620:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763077.884642:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763077.884654:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763077/real 1789763077] req@ffff8d25d1b09c00 x1876699507741056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763077.884669:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507741056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763077.884678:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763077.884682:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763077.884685:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763077.884713:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763077.884734:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763077.884746:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763077/real 1789763077] req@ffff8d25d1b08700 x1876699507741184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763077.884761:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507741184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763102 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763077.884771:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763077.884775:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763077.884778:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741184:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763077.947606:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763078.971635:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763079.995623:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763081.019657:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763082.043626:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763083.003538:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763083.003556:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763083.003564:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763083.003581:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3060 00000100:00080000:3.0:1789763083.003600:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763083.003611:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763083.003686:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507741312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763083.003746:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763083.003753:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763083.003756:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763083.003763:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3060 00000100:00080000:3.0:1789763083.003771:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763083.003780:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763083.003797:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763083.003818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507741312:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763083.003821:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507741440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763083.003862:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763083.003867:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763083.003870:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763083.003878:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3060 00000100:00080000:3.0:1789763083.003884:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763083.003890:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763083.003897:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763083.003913:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507741568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763083.003927:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763083.003939:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763083.003942:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763083.003949:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763083.003961:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3060 00000100:00080000:3.0:1789763083.003971:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763083.003981:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763083.003989:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763083.004012:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507741696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763083.004028:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763083] req@ffff8d25d1b0bb80 x1876699507741312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763083.004040:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763083.004081:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763083.004085:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763083.004099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507741312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763083.004100:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3060 00000100:00080000:3.0:1789763083.004114:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763083.004128:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763083.004140:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763083.004173:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507741824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763083.004175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763083.004197:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763083.004225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507741312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763083.004262:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763083.004267:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763083.004271:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763083.004277:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3060 00000100:00080000:3.0:1789763083.004282:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763083.004292:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507741440:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763083.004302:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763083.004314:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763083.004338:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507741952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763083.004359:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763083.004424:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763083] req@ffff8d25d1b08e00 x1876699507741440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763083.004443:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507741440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763083.004459:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763083.004466:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763083.004473:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507741440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763083.004486:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507741568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763083.004549:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763083.004573:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763083] req@ffff8d25d1b09880 x1876699507741568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763083.004593:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507741568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763083.004613:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763083.004617:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763083.004621:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507741568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763083.004629:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507741696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763083.004657:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763083.004670:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763083] req@ffff8d25d1b0b800 x1876699507741696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763083.004684:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507741696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763083.004694:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763083.004698:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763083.004702:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507741696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763083.004708:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763083.004728:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763083.004741:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763083] req@ffff8d25d1b0a680 x1876699507741824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763083.004755:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507741824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763083.004764:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763083.004768:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763083.004772:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763083.004803:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763083.004823:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763083.004836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763083/real 1789763083] req@ffff8d25d1b0bb80 x1876699507741952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763083.004851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507741952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763108 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763083.004860:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763083.004863:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763083.004867:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507741952:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763083.067677:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763084.091629:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763085.115700:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763086.139617:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763087.163599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763088.123559:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763088.123576:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763088.123584:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763088.123601:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3065 00000100:00080000:3.0:1789763088.123616:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763088.123628:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763088.123690:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507742080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763088.123731:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763088.123737:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763088.123746:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763088.123753:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3065 00000100:00080000:3.0:1789763088.123759:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763088.123765:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763088.123781:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763088.123799:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507742208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763088.123815:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763088.123821:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507742080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763088.123822:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763088.123827:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763088.123841:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3065 00000100:00080000:3.0:1789763088.123846:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763088.123851:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763088.123858:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763088.123876:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507742336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763088.123889:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763088.123894:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763088.123897:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763088.123901:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3065 00000100:00080000:3.0:1789763088.123905:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763088.123909:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763088.123915:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763088.123929:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507742464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763088.123951:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763088.123956:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763088.123958:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763088.123971:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3065 00000800:00000100:2.0:1789763088.123973:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763088.123978:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763088.123987:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763088.123994:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763088.124011:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507742592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763088.124023:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763088.124028:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763088.124031:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763088.124036:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3065 00000100:00080000:3.0:1789763088.124041:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763088.124048:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763088.124054:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763088.124067:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffff8d25d1b08700 x1876699507742080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763088.124073:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507742720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763088.124127:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507742080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.124147:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.124154:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763088.124164:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507742080:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763088.124209:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763088.124247:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763088.124266:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffff8d25d1b09c00 x1876699507742208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.124298:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507742208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.124311:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.124316:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763088.124321:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763088.124329:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763088.124355:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763088.124369:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffff8d25d1b0aa00 x1876699507742336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.124432:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507742336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.124444:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.124448:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763088.124452:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763088.124496:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742464:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763088.124543:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763088.124569:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffff8d25d1b09180 x1876699507742464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.124592:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507742464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.124603:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.124607:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763088.124611:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763088.124619:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507742592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763088.124648:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763088.124663:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffff8d25d1b0ad80 x1876699507742592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.124678:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507742592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.124687:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.124691:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763088.124694:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507742592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763088.124702:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507742720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763088.124721:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763088.124733:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763088/real 1789763088] req@ffff8d25d1b08a80 x1876699507742720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763088.124747:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507742720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763113 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763088.124766:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763088.124771:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763088.124775:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507742720:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763088.187621:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763089.211668:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763090.235635:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763091.259602:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763092.283665:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763093.243560:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763093.243574:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763093.243580:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763093.243595:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3070 00000100:00080000:3.0:1789763093.243624:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763093.243635:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763093.243688:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507742848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763093.243723:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763093.243727:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763093.243729:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763093.243734:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3070 00000100:00080000:3.0:1789763093.243738:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763093.243743:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763093.243756:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763093.243772:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507742976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763093.243782:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507742848:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763093.243786:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763093.243793:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763093.243795:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763093.243801:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3070 00000100:00080000:3.0:1789763093.243805:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763093.243808:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763093.243813:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763093.243827:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507743104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763093.243838:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763093.243843:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763093.243845:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763093.243849:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3070 00000100:00080000:3.0:1789763093.243865:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763093.243870:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763093.243875:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763093.243885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763093.243888:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507743232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763093.243900:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763093.243903:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763093.243906:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763093.243911:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3070 00000100:00080000:3.0:1789763093.243916:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763093.243920:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763093.243924:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763093.243935:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507743360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763093.243946:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffff8d25d1b0b480 x1876699507742848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.243967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507742848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.243978:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.243982:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763093.243987:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507742848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763093.244016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763093.244042:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763093.244051:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763093.244061:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffff8d25d1b0a300 x1876699507742976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.244073:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507742976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.244088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.244091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763093.244094:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507742976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763093.244099:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763093.244115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763093.244125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffff8d25d1b08000 x1876699507743104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.244139:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507743104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.244146:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.244149:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763093.244152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763093.244157:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763093.244174:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763093.244185:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763093.244186:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffff8d25d1b09500 x1876699507743232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763093.244191:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763093.244203:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3070 00000100:00100000:2.0:1789763093.244210:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507743232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763093.244210:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763093.244223:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.244227:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763093.244231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743232:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789763093.244259:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763093.244264:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507743360:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789763093.244269:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763093.244288:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507743488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763093.244295:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763093.244323:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffff8d25d1b08380 x1876699507743360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.244339:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507743360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.244351:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.244355:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763093.244359:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507743360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763093.244415:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507743488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763093.244460:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763093.244473:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763093/real 1789763093] req@ffff8d25d1b0b100 x1876699507743488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763093.244484:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507743488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763118 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763093.244491:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763093.244494:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763093.244497:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507743488:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763093.307609:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763094.331611:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763095.355623:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763096.379565:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763097.403624:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763098.363593:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763098.363610:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763098.363617:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763098.363638:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3076 00000100:00080000:2.0:1789763098.363656:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763098.363668:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763098.363730:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507743616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763098.363774:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763098.363782:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763098.363786:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763098.363795:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3076 00000100:00080000:2.0:1789763098.363804:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763098.363815:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763098.363836:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763098.363851:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507743616:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763098.363870:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507743744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763098.363904:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763098.363911:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763098.363913:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763098.363919:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3076 00000100:00080000:2.0:1789763098.363944:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763098.363952:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763098.363961:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763098.363971:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763098.364003:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507743872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763098.364043:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763098.364050:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763098.364053:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763098.364062:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3076 00000100:00080000:2.0:1789763098.364071:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763098.364080:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763098.364089:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffff8d25c8021880 x1876699507743616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763098.364095:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763098.364124:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507744000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763098.364139:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763098.364141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507743616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763098.364148:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763098.364153:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763098.364165:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3076 00000100:00080000:2.0:1789763098.364176:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763098.364188:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789763098.364194:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789763098.364195:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763098.364213:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763098.364218:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507744128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763098.364230:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507743616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763098.364240:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763098.364246:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763098.364250:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763098.364255:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3076 00000100:00080000:2.0:1789763098.364259:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763098.364264:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763098.364269:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763098.364283:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763098.364301:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507744256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763098.364355:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763098.364478:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffff8d25c8021c00 x1876699507743744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763098.364500:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507743744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763098.364512:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763098.364517:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763098.364523:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763098.364532:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763098.364564:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763098.364579:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffff8d25c8021500 x1876699507743872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763098.364594:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507743872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763098.364604:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763098.364608:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763098.364612:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507743872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763098.364620:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763098.364641:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763098.364653:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffff8d25c8020e00 x1876699507744000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763098.364667:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507744000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763098.364677:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763098.364681:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763098.364685:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763098.364696:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507744128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763098.364730:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763098.364755:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffff8d25c8020700 x1876699507744128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763098.364782:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507744128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763098.364800:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763098.364804:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763098.364809:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507744128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763098.364835:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507744256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763098.364865:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763098.364879:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763098/real 1789763098] req@ffff8d25c8021880 x1876699507744256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763098.364895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507744256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763123 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763098.364905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763098.364909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763098.364913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507744256:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763098.427647:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763099.451657:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763100.475636:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763101.499676:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763102.523667:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763103.483607:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763103.483621:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763103.483627:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763103.483640:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3081 00000100:00080000:3.0:1789763103.483653:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763103.483663:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763103.483714:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507744384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763103.483759:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763103.483764:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763103.483766:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763103.483772:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3081 00000100:00080000:3.0:1789763103.483780:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763103.483785:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763103.483796:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763103.483818:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507744512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763103.483850:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507744384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763103.483872:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763103.483883:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763103.483888:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763103.483901:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3081 00000100:00080000:3.0:1789763103.483910:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763103.483920:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763103.483929:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763103.483953:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507744640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763103.483968:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763103.483974:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763103.483979:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763103.483991:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763103.484009:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3081 00000100:00080000:3.0:1789763103.484021:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763103.484028:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763103.484038:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763103.484058:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507744768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763103.484074:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763103/real 1789763103] req@ffff8d25d1b0b100 x1876699507744384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763103.484077:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763103.484083:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763103.484085:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763103.484092:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3081 00000100:00080000:3.0:1789763103.484110:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763103.484126:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763103.484144:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763103.484177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507744384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763103.484179:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507744896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763103.484301:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763103.484306:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763103.484309:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763103.484311:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763103.484321:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3081 00000100:00080000:2.0:1789763103.484332:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763103.484334:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763103.484425:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507744384:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763103.484442:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763103.484455:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763103.484485:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507745024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763103.484493:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763103.484573:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763103.484606:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763103/real 1789763103] req@ffff8d25d1b08380 x1876699507744512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763103.484635:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507744512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763103.484647:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763103.484651:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763103.484656:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763103.484664:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763103.484689:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763103.484703:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763103/real 1789763103] req@ffff8d25d1b09500 x1876699507744640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763103.484718:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507744640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763103.484728:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763103.484732:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763103.484735:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763103.484742:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763103.484764:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763103.484776:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763103/real 1789763103] req@ffff8d25d1b08000 x1876699507744768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763103.484790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507744768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763103.484809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763103.484813:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763103.484817:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507744768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763103.484824:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507744896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763103.484844:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763103.484855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763103/real 1789763103] req@ffff8d25d1b0a300 x1876699507744896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763103.484870:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507744896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763103.484880:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763103.484884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763103.484887:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507744896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763103.484919:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507745024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763103.484940:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763103.484953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763103/real 1789763103] req@ffff8d25d1b0b480 x1876699507745024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763103.484966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507745024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763128 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763103.484976:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763103.484980:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763103.484984:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507745024:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763103.547665:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763104.571582:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763105.595635:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763106.619533:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763107.643608:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763108.603516:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763108.603529:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763108.603535:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763108.603559:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3086 00000100:00080000:0.0:1789763108.603573:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763108.603583:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763108.603647:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507745152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763108.603679:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763108.603684:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763108.603687:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763108.603692:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3086 00000100:00080000:0.0:1789763108.603699:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763108.603707:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763108.603723:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763108.603748:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507745152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763108.603754:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507745280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763108.603785:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763108.603792:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763108.603796:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763108.603804:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3086 00000100:00080000:0.0:1789763108.603813:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763108.603821:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763108.603832:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763108.603850:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507745408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763108.603865:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763108.603873:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763108.603875:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763108.603881:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763108.603889:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3086 00000100:00080000:0.0:1789763108.603895:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763108.603901:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763108.603909:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763108.603925:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507745536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763108.603939:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763108.603943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763108/real 1789763108] req@ffff8d25d25bb480 x1876699507745152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763108.603945:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763108.603951:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763108.603965:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3086 00000100:00080000:0.0:1789763108.603976:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763108.603981:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763108.603988:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763108.603989:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507745152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763108.604006:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507745664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763108.604008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763108.604025:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763108.604039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507745152:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763108.604042:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763108.604052:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763108.604056:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763108.604062:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3086 00000100:00080000:0.0:1789763108.604067:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763108.604071:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763108.604078:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763108.604084:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507745280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763108.604096:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507745792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763108.604127:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763108.604149:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763108/real 1789763108] req@ffff8d25d25b8e00 x1876699507745280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763108.604166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507745280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763108.604177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763108.604181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763108.604185:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507745280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763108.604193:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507745408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763108.604213:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763108.604225:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763108/real 1789763108] req@ffff8d25d25b9c00 x1876699507745408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763108.604239:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507745408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763108.604249:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763108.604253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763108.604256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507745408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763108.604263:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507745536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763108.604282:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763108.604293:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763108/real 1789763108] req@ffff8d25d25bb100 x1876699507745536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763108.604609:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507745536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763108.604629:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763108.604634:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763108.604638:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507745536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763108.604648:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507745664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763108.604680:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763108.604696:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763108/real 1789763108] req@ffff8d25d25b8700 x1876699507745664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763108.604720:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507745664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763108.604735:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763108.604742:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763108.604749:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507745664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763108.604784:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507745792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763108.604820:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763108.604836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763108/real 1789763108] req@ffff8d25d25b8380 x1876699507745792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763108.604851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507745792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763133 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763108.604862:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763108.604866:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763108.604869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507745792:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763108.667620:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763109.691606:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763110.715590:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763111.739556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763112.763597:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763113.723532:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763113.723549:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763113.723556:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763113.723574:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3091 00000100:00080000:3.0:1789763113.723590:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763113.723602:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763113.723667:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507745920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763113.723726:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763113.723734:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763113.723739:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763113.723752:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3091 00000100:00080000:3.0:1789763113.723763:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763113.723771:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763113.723789:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763113.723796:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507745920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763113.723817:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507746048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763113.723839:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763113.723845:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763113.723850:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763113.723857:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3091 00000100:00080000:3.0:1789763113.723880:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763113.723887:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763113.723897:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763113.723909:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763113.723917:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507746176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763113.723933:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763113.723941:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763113.723945:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763113.723955:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3091 00000100:00080000:3.0:1789763113.723964:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763113.723973:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763113.723987:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763113.723991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763113/real 1789763113] req@ffff8d25d1b08a80 x1876699507745920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763113.724023:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507746304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763113.724062:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763113.724066:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763113.724069:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763113.724076:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3091 00000100:00080000:3.0:1789763113.724083:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763113.724099:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763113.724102:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507745920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763113.724108:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763113.724139:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507746432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763113.724150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763113.724155:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763113.724165:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763113.724169:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763113.724173:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763113.724179:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3091 00000100:00080000:3.0:1789763113.724185:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763113.724189:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763113.724194:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763113.724210:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507745920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763113.724213:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507746560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763113.724307:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763113.724352:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763113.724424:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763113/real 1789763113] req@ffff8d25d1b0ad80 x1876699507746048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763113.724455:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507746048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763113.724474:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763113.724481:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763113.724486:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763113.724496:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763113.724530:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763113.724547:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763113/real 1789763113] req@ffff8d25d1b09180 x1876699507746176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763113.724563:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507746176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763113.724581:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763113.724585:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763113.724589:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763113.724597:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763113.724617:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763113.724629:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763113/real 1789763113] req@ffff8d25d1b0aa00 x1876699507746304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763113.724643:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507746304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763113.724653:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763113.724657:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763113.724661:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763113.724668:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507746432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763113.724688:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763113.724700:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763113/real 1789763113] req@ffff8d25d1b09c00 x1876699507746432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763113.724714:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507746432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763113.724723:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763113.724727:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763113.724732:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507746432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763113.724739:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507746560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763113.724758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763113.724770:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763113/real 1789763113] req@ffff8d25d1b08700 x1876699507746560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763113.724783:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507746560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763138 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763113.724792:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763113.724796:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763113.724800:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507746560:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763113.787592:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763114.811608:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763115.835585:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763116.859587:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763117.883631:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763118.843588:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763118.843608:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763118.843616:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763118.843635:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3096 00000100:00080000:0.0:1789763118.843652:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763118.843666:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763118.843719:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507746688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763118.843775:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763118.843779:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763118.843781:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763118.843788:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3096 00000100:00080000:0.0:1789763118.843795:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763118.843804:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763118.843820:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763118.843858:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507746816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763118.843868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507746688:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763118.843905:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763118.843911:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763118.843914:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763118.843923:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3096 00000100:00080000:0.0:1789763118.843929:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763118.843934:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763118.843942:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763118.843978:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507746944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763118.843989:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763118.843993:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763118.843995:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763118.843998:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3096 00000100:00080000:0.0:1789763118.844003:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763118.844013:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789763118.844015:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763118.844033:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763118.844052:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507747072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763118.844064:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763118.844068:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763118.844070:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763118.844074:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3096 00000100:00080000:0.0:1789763118.844082:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763118.844088:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763118.844095:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763118.844104:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763118/real 1789763118] req@ffff8d25d25b9f80 x1876699507746688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763118.844113:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507747200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763118.844174:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763118.844179:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763118.844181:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763118.844189:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507746688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763118.844195:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3096 00000100:00080000:0.0:1789763118.844210:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763118.844254:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763118.844264:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763118.844265:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763118.844277:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763118.844287:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507746688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763118.844290:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507747328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763118.844369:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763118.844443:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763118.844466:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763118/real 1789763118] req@ffff8d25d25bb800 x1876699507746816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763118.844494:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507746816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763118.844510:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763118.844517:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763118.844524:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763118.844538:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763118.844581:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763118.844604:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763118/real 1789763118] req@ffff8d25d25ba680 x1876699507746944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763118.844627:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507746944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763118.844645:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763118.844652:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763118.844658:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507746944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763118.844672:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763118.844715:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763118.844739:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763118/real 1789763118] req@ffff8d25d25bbb80 x1876699507747072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763118.844764:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507747072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763118.844775:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763118.844779:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763118.844784:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763118.844792:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507747200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763118.844822:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763118.844836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763118/real 1789763118] req@ffff8d25d25b9180 x1876699507747200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763118.844851:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507747200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763118.844862:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763118.844865:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763118.844869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507747200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763118.844876:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507747328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763118.844896:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763118.844907:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763118/real 1789763118] req@ffff8d25d25b8a80 x1876699507747328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763118.844921:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507747328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763143 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763118.844939:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763118.844944:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763118.844948:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507747328:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763118.907636:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763119.931583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763120.955601:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763121.979654:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763123.003603:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763123.963618:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763123.963635:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763123.963643:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763123.963660:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3101 00000100:00080000:2.0:1789763123.963677:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763123.963688:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763123.963752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507747456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763123.963822:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763123.963831:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763123.963835:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763123.963846:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3101 00000100:00080000:2.0:1789763123.963855:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763123.963867:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763123.963881:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507747456:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763123.963895:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763123.963932:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507747584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763123.963953:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763123.963958:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763123.963962:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763123.963974:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3101 00000800:00000100:3.0:1789763123.963977:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763123.963987:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763123.963998:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763123.964007:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763123.964024:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507747712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763123.964036:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763123.964041:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763123.964049:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763123.964050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763123/real 1789763123] req@ffff8d25c8023480 x1876699507747456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763123.964075:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3101 00000100:00080000:2.0:1789763123.964091:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763123.964100:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763123.964106:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763123.964124:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507747840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763123.964126:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507747456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763123.964219:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763123.964244:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763123.964245:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763123.964260:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763123.964264:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763123.964272:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507747456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763123.964276:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3101 00000100:00080000:2.0:1789763123.964289:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763123.964300:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763123.964316:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763123.964334:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507747968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763123.964350:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747584:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763123.964352:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763123.964371:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763123.964373:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763123.964450:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763123.964462:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3101 00000100:00080000:2.0:1789763123.964471:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763123.964478:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763123/real 1789763123] req@ffff8d25c8023100 x1876699507747584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763123.964487:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763123.964510:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763123.964521:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507747584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763123.964534:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507748096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763123.964542:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763123.964560:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763123.964573:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763123.964590:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763123.964618:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763123.964630:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763123/real 1789763123] req@ffff8d25c8022d80 x1876699507747712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763123.964642:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507747712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763123.964650:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763123.964653:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763123.964656:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763123.964662:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763123.964678:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763123.964688:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763123/real 1789763123] req@ffff8d25c8020380 x1876699507747840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763123.964700:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507747840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763123.964708:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763123.964711:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763123.964714:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507747840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763123.964730:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507747968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763123.964746:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763123.964757:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763123/real 1789763123] req@ffff8d25c8022300 x1876699507747968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763123.964768:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507747968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763123.964776:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763123.964779:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763123.964782:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507747968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763123.964787:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507748096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763123.964802:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763123.964812:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763123/real 1789763123] req@ffff8d25c8023b80 x1876699507748096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763123.964823:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507748096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763148 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763123.964838:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763123.964841:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763123.964844:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507748096:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763124.027636:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763125.051597:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763126.075568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763127.099636:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763128.123560:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763129.083515:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763129.083525:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763129.083530:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763129.083542:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3106 00000100:00080000:2.0:1789763129.083560:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763129.083570:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763129.083614:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507748224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763129.083648:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763129.083669:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763129.083672:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763129.083679:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3106 00000100:00080000:2.0:1789763129.083686:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763129.083692:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763129.083702:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507748224:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763129.083706:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763129.083733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507748352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763129.083746:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763129.083750:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763129.083753:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763129.083759:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3106 00000100:00080000:2.0:1789763129.083765:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763129.083772:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763129.083780:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763129.083798:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507748480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763129.083816:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507748352:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763129.083816:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763129.083847:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763129.083850:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763129.083859:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3106 00000100:00080000:2.0:1789763129.083866:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763129.083871:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763129.083877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507748480:192.168.203.141@tcp:38:kworker.0 00000008:00100000:2.0:1789763129.083881:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763129.083901:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507748608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763129.083933:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763129.083937:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763129.083940:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763129.083947:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3106 00000100:00080000:2.0:1789763129.083952:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763129.083957:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763129.083961:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763129.083962:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507748608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763129.084004:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507748736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763129.084024:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763129.084029:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763129.084032:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763129.084036:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507748736:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763129.084046:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3106 00000100:00080000:2.0:1789763129.084054:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763129.084061:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763129.084071:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763129.084101:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507748864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763129.084194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507748864:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763129.147580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763130.171627:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763131.195565:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763132.155823:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789763132.155911:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763132.155942:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763132.156025:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763132.156046:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763132.156062:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763132.156076:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763132.156093:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789763132.156181:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763132] req@ffff8d25c8022a00 x1876699507748864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763132.156246:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507748864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763132.156280:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763132.156291:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763132.156299:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507748864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763132.156314:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763132] req@ffff8d25c8020000 x1876699507748736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763132.156338:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507748736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763132.156354:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763132.156360:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763132.156367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507748736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763132.156433:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763132] req@ffff8d25c8020a80 x1876699507748608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763132.156468:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507748608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763132.156489:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763132.156500:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763132.156509:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507748608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763132.156525:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763132] req@ffff8d25c8021180 x1876699507748480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763132.156553:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507748480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763132.156569:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763132.156578:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763132.156584:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507748480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763132.156597:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763132] req@ffff8d25c8022680 x1876699507748352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763132.156622:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507748352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763132.156638:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763132.156645:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763132.156651:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507748352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763132.156665:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763129/real 1789763132] req@ffff8d25c8023800 x1876699507748224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763132.156688:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507748224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763154 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763132.156702:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763132.156708:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763132.156714:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507748224:192.168.203.141@tcp:250:kworker.0 00010000:00010000:2.0:1789763132.219639:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763133.243579:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763134.203559:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763134.203576:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763134.203583:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763134.203599:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3111 00000100:00080000:2.0:1789763134.203614:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763134.203626:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763134.203677:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507748992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763134.203715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763134.203720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763134.203723:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763134.203729:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3111 00000100:00080000:2.0:1789763134.203734:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763134.203740:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763134.203753:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763134.203771:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507749120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763134.203779:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507748992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763134.203798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763134.203804:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763134.203807:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763134.203815:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3111 00000100:00080000:2.0:1789763134.203820:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763134.203825:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763134.203832:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763134.203850:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507749248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763134.203863:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763134.203868:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763134.203870:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763134.203882:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3111 00000800:00000100:3.0:1789763134.203888:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763134.203891:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763134.203900:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763134.203907:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763134.203923:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507749376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763134.203937:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763134.203943:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763134.203947:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763134.203955:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3111 00000100:00080000:2.0:1789763134.203964:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763134.203975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffff8d25c8023800 x1876699507748992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763134.203981:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763134.203999:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763134.204036:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507749504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763134.204042:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507748992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763134.204079:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763134.204101:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763134.204109:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763134.204138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763134.204150:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3111 00000100:00080000:3.0:1789763134.204159:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763134.204166:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763134.204176:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507748992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763134.204186:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763134.204211:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763134.204238:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507749632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763134.204244:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763134.204318:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763134.204348:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffff8d25c8022680 x1876699507749120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.204373:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507749120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.204432:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.204439:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763134.204446:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763134.204457:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763134.204500:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763134.204525:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffff8d25c8021180 x1876699507749248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.204549:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507749248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.204564:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.204570:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763134.204576:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763134.204588:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763134.204625:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763134.204645:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffff8d25c8020a80 x1876699507749376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.204665:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507749376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.204680:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.204686:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763134.204692:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763134.204705:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507749504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763134.204743:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763134.204915:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffff8d25c8020000 x1876699507749504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.204948:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507749504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.204966:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.204972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763134.204979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507749504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763134.205017:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507749632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763134.205064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763134.205088:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763134/real 1789763134] req@ffff8d25c8022a00 x1876699507749632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763134.205112:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507749632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763159 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763134.205153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763134.205160:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763134.205166:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507749632:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763134.267612:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763135.291489:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763136.315607:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763137.339626:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763138.363658:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763139.323569:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763139.323581:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763139.323587:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.323602:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3117 00000100:00080000:3.0:1789763139.323617:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763139.323629:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763139.323702:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507749760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763139.323747:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763139.323753:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763139.323757:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.323765:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3117 00000100:00080000:3.0:1789763139.323773:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763139.323781:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763139.323796:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763139.323818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507749760:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763139.323827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507749888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763139.323851:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763139.323858:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763139.323862:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.323883:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3117 00000100:00080000:3.0:1789763139.323892:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763139.323900:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763139.323907:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763139.323918:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763139.323936:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507750016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763139.323951:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763139.323959:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffff8d25d1b0bb80 x1876699507749760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763139.323960:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763139.323966:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.323979:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3117 00000100:00080000:3.0:1789763139.323988:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763139.324014:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763139.324023:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763139.324032:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507749760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763139.324047:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507750144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763139.324058:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763139.324069:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763139.324073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763139.324125:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763139.324126:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507749760:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763139.324133:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.324146:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3117 00000100:00080000:3.0:1789763139.324154:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763139.324163:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763139.324185:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749888:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789763139.324194:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763139.324224:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507750272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763139.324238:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763139.324254:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763139.324268:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffff8d25d1b0a680 x1876699507749888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763139.324282:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763139.324290:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763139.324319:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3117 00000100:00080000:3.0:1789763139.324327:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763139.324328:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507749888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763139.324339:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763139.324353:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763139.324356:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763139.324364:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763139.324370:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507749888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763139.324377:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507750400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763139.324405:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763139.324486:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763139.324509:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffff8d25d1b0b800 x1876699507750016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763139.324526:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507750016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763139.324537:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763139.324541:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763139.324544:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763139.324552:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763139.324573:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763139.324586:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffff8d25d1b09880 x1876699507750144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763139.324624:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507750144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763139.324638:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763139.324644:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763139.324650:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763139.324683:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507750272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763139.324718:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763139.324739:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffff8d25d1b08e00 x1876699507750272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763139.324760:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507750272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763139.324787:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763139.324793:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763139.324799:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507750272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763139.324807:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507750400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763139.324832:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763139.324845:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763139/real 1789763139] req@ffff8d25d1b0bb80 x1876699507750400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763139.324858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507750400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763164 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763139.324867:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763139.324870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763139.324874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507750400:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763139.387598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763140.411622:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763141.435675:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763142.459590:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763143.483639:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763144.443571:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763144.443586:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763144.443592:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763144.443618:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3122 00000100:00080000:0.0:1789763144.443629:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763144.443639:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763144.443682:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507750528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763144.443715:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763144.443720:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763144.443723:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763144.443728:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3122 00000100:00080000:0.0:1789763144.443733:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763144.443738:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763144.443750:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763144.443768:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507750656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763144.443783:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507750528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763144.443788:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763144.443799:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763144.443801:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763144.443810:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3122 00000100:00080000:0.0:1789763144.443819:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763144.443827:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763144.443839:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763144.443865:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507750784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763144.443889:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763144.443913:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763144.443919:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763144.443922:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763144.443931:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3122 00000100:00080000:0.0:1789763144.443937:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763144.443943:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763144.443952:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763144.443980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763144/real 1789763144] req@ffff8d25d25bad80 x1876699507750528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763144.443988:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507750912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763144.444070:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763144.444079:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763144.444080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507750528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763144.444083:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763144.444099:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3122 00000100:00080000:0.0:1789763144.444110:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763144.444116:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763144.444120:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763144.444128:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789763144.444132:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763144.444140:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507750528:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763144.444158:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507751040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763144.444174:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763144.444179:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763144.444184:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750656:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763144.444185:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763144.444199:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3122 00000100:00080000:0.0:1789763144.444205:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763144.444211:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763144.444218:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763144.444241:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763144.444249:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507751168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763144.444266:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763144/real 1789763144] req@ffff8d25d25ba300 x1876699507750656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763144.444288:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507750656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763144.444300:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763144.444306:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763144.444310:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763144.444317:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763144.444339:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763144.444352:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763144/real 1789763144] req@ffff8d25d25b8000 x1876699507750784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763144.444367:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507750784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763144.444410:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763144.444414:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763144.444418:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763144.444426:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763144.444451:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763144.444467:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763144/real 1789763144] req@ffff8d25d25b8a80 x1876699507750912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763144.444481:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507750912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763144.444492:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763144.444495:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763144.444499:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507750912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763144.444505:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763144.444524:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763144.444535:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763144/real 1789763144] req@ffff8d25d25b9180 x1876699507751040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763144.444550:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507751040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763144.444565:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763144.444572:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763144.444578:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763144.444611:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763144.444654:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763144.444677:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763144/real 1789763144] req@ffff8d25d25bbb80 x1876699507751168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763144.444692:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507751168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763169 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763144.444704:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763144.444708:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763144.444712:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751168:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763144.507604:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763145.531646:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763146.555678:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763147.579713:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763148.603541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763149.563561:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763149.563583:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763149.563588:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763149.563599:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3127 00000100:00080000:3.0:1789763149.563609:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763149.563617:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763149.563657:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507751296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763149.563687:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763149.563692:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763149.563694:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763149.563699:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3127 00000100:00080000:3.0:1789763149.563703:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763149.563708:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763149.563718:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763149.563738:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507751424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763149.563740:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507751296:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763149.563763:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763149.563767:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763149.563770:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763149.563775:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3127 00000100:00080000:3.0:1789763149.563779:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763149.563784:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763149.563789:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763149.563804:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507751552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763149.563822:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763149.563824:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763149.563834:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763149.563838:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763149.563847:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3127 00000100:00080000:3.0:1789763149.563855:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763149.563864:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763149.563875:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763149.563876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763149/real 1789763149] req@ffff8d25d1b08700 x1876699507751296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763149.563907:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507751680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763149.563909:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507751296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763149.563972:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763149.563974:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763149.563984:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763149.563986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763149.563990:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763149.563997:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507751296:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763149.564029:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3127 00000100:00080000:3.0:1789763149.564058:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763149.564067:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763149.564078:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763149.564082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507751424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763149.564126:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507751808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763149.564145:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763149.564159:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763149.564165:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763149.564168:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763149.564173:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763149/real 1789763149] req@ffff8d25d1b09c00 x1876699507751424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763149.564180:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3127 00000100:00080000:3.0:1789763149.564192:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763149.564201:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763149.564207:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507751424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763149.564213:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763149.564230:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763149.564237:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763149.564244:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507751936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763149.564247:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507751424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763149.564269:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507751552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763149.564301:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763149.564316:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763149/real 1789763149] req@ffff8d25d1b0aa00 x1876699507751552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763149.564331:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507751552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763149.564341:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763149.564344:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763149.564348:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507751552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763149.564355:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507751680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763149.564374:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763149.564385:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763149/real 1789763149] req@ffff8d25d1b09180 x1876699507751680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763149.564399:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507751680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763149.564437:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763149.564442:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763149.564446:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507751680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763149.564471:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763149.564496:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763149.564509:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763149/real 1789763149] req@ffff8d25d1b0ad80 x1876699507751808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763149.564525:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507751808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763149.564563:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763149.564566:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763149.564570:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763149.564575:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763149.564593:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763149.564603:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763149/real 1789763149] req@ffff8d25d1b08a80 x1876699507751936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763149.564616:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507751936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763174 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763149.564624:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763149.564626:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763149.564629:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507751936:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763149.627677:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763150.651625:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763151.675627:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763152.699568:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763153.723558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763154.683619:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763154.683638:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763154.683647:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763154.683669:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3132 00000100:00080000:2.0:1789763154.683690:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763154.683706:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763154.683782:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507752064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763154.683829:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763154.683835:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763154.683838:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763154.683844:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3132 00000100:00080000:2.0:1789763154.683850:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763154.683857:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763154.683873:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763154.683896:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507752192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763154.683911:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507752064:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763154.683926:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763154.683934:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763154.683936:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763154.683944:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3132 00000100:00080000:2.0:1789763154.683949:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763154.683955:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763154.683961:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763154.684029:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507752320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763154.684059:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763154.684082:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763154.684088:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763154.684091:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763154.684100:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3132 00000100:00080000:2.0:1789763154.684107:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763154.684114:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763154.684123:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763154.684142:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507752448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763154.684157:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763154.684161:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763154.684163:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763154.684177:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3132 00000100:00100000:1.0:1789763154.684182:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763154/real 1789763154] req@ffff8d25c8023b80 x1876699507752064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763154.684185:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763154.684192:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763154.684203:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763154.684228:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507752576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763154.684245:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763154.684249:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763154.684255:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507752064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763154.684256:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763154.684285:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3132 00000100:00080000:2.0:1789763154.684304:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763154.684318:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763154.684329:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763154.684356:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507752704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763154.684366:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763154.684443:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763154.684451:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507752064:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763154.684503:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763154.684553:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763154.684587:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763154/real 1789763154] req@ffff8d25c8022300 x1876699507752192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763154.684617:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507752192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763154.684635:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763154.684643:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763154.684647:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763154.684657:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763154.684689:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763154.684705:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763154/real 1789763154] req@ffff8d25c8020380 x1876699507752320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763154.684721:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507752320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763154.684732:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763154.684736:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763154.684740:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763154.684749:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752448:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763154.684782:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763154.684805:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763154/real 1789763154] req@ffff8d25c8022d80 x1876699507752448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763154.684831:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507752448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763154.684849:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763154.684856:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763154.684862:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763154.684872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507752576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763154.684901:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763154.684917:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763154/real 1789763154] req@ffff8d25c8023100 x1876699507752576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763154.684932:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507752576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763154.684943:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763154.684947:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763154.684951:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507752576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763154.684957:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507752704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763154.685014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763154.685030:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763154/real 1789763154] req@ffff8d25c8023480 x1876699507752704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763154.685046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507752704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763179 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763154.685057:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763154.685061:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763154.685065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507752704:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763154.747656:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763155.771634:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763156.795668:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763157.819589:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763158.843637:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763159.803502:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763159.803519:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763159.803526:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763159.803543:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3137 00000100:00080000:2.0:1789763159.803571:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763159.803584:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763159.803645:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507752832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763159.803684:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763159.803689:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763159.803692:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763159.803698:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3137 00000100:00080000:2.0:1789763159.803703:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763159.803709:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763159.803725:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763159.803749:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507752960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763159.803751:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507752832:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763159.803781:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763159.803785:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763159.803788:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763159.803793:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3137 00000100:00080000:2.0:1789763159.803798:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763159.803803:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763159.803809:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763159.803828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507753088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763159.803840:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763159.803845:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763159.803848:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763159.803852:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3137 00000100:00080000:2.0:1789763159.803856:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763159.803861:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763159.803868:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763159.803883:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507753216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763159.803891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763159.803909:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763159.803915:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763159.803918:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763159.803924:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3137 00000100:00080000:2.0:1789763159.803967:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763159.803975:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763159.803984:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763159.803997:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763159/real 1789763159] req@ffff8d25c8021880 x1876699507752832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763159.804008:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507753344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763159.804036:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763159.804042:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763159.804047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507752832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763159.804048:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763159.804064:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3137 00000100:00080000:2.0:1789763159.804076:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763159.804086:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763159.804087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789763159.804093:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763159.804100:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763159.804110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507752832:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763159.804118:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507753472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763159.804154:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763159.804190:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763159.804208:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763159/real 1789763159] req@ffff8d25c8020700 x1876699507752960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763159.804224:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507752960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763159.804244:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763159.804249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763159.804253:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507752960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763159.804260:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763159.804280:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763159.804292:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763159/real 1789763159] req@ffff8d25c8020e00 x1876699507753088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763159.804307:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507753088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763159.804318:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763159.804321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763159.804324:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763159.804331:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763159.804351:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763159.804363:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763159/real 1789763159] req@ffff8d25c8021500 x1876699507753216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763159.804411:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507753216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763159.804422:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763159.804426:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763159.804432:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763159.804440:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507753344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763159.804465:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763159.804478:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763159/real 1789763159] req@ffff8d25c8021c00 x1876699507753344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763159.804493:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507753344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763159.804503:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763159.804506:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763159.804510:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507753344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763159.804517:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507753472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763159.804536:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763159.804548:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763159/real 1789763159] req@ffff8d25c8023480 x1876699507753472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763159.804563:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507753472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763184 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763159.804572:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763159.804575:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763159.804579:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507753472:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763159.867598:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763160.891599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763161.915591:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763162.939644:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763163.963655:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763164.923659:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763164.923674:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763164.923683:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763164.923701:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3142 00000100:00080000:3.0:1789763164.923719:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763164.923731:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763164.923792:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507753600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763164.923836:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763164.923842:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763164.923846:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763164.923854:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3142 00000100:00080000:3.0:1789763164.923860:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763164.923866:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763164.923882:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763164.923945:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507753600:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763164.923946:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507753728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763164.923991:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763164.924001:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763164.924006:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763164.924015:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3142 00000100:00080000:3.0:1789763164.924026:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763164.924037:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763164.924050:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763164.924079:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763164.924138:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763164/real 1789763164] req@ffff8d25d1b0b480 x1876699507753600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763164.924166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507753600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763164.924183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763164.924189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763164.924195:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507753600:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763164.924238:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763164.924272:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763164.924291:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763164/real 1789763164] req@ffff8d25d1b0a300 x1876699507753728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763164.924306:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507753728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763164.924316:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763164.924321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763164.924327:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763164.924489:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507753856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763164.924590:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763164.924595:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763164.924601:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763164.924612:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763164.924624:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3142 00000100:00080000:3.0:1789763164.924635:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763164.924643:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763164.924657:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789763164.924674:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763164.924689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507753984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763164.924715:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763164.924716:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763164/real 1789763164] req@ffff8d25d1b08000 x1876699507753856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763164.924747:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763164.924759:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763164.924784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3142 00000100:00080000:3.0:1789763164.924791:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763164.924793:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507753856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763164.924803:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763164.924829:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763164.924833:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763164.924843:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763164.924855:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507754112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763164.924862:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763164.924936:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763164.924941:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763164.924945:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763164.924951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753984:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789763164.924962:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3142 00000100:00080000:3.0:1789763164.924975:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763164.924986:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763164.924998:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763164.925017:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763164.925034:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507754240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763164.925054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763164/real 1789763164] req@ffff8d25d1b09500 x1876699507753984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763164.925099:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507753984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763164.925115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763164.925121:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763164.925130:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507753984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763164.925141:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507754112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763164.925171:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763164.925187:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763164/real 1789763164] req@ffff8d25d1b08380 x1876699507754112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763164.925201:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507754112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763164.925211:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763164.925214:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763164.925218:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507754112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763164.925241:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507754240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763164.925262:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763164.925275:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763164/real 1789763164] req@ffff8d25d1b0b100 x1876699507754240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763164.925290:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507754240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763189 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763164.925299:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763164.925302:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763164.925306:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507754240:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763164.987581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763166.011489:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763167.035522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763168.059577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763169.083644:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763170.043599:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763170.043612:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763170.043617:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763170.043630:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3147 00000100:00080000:3.0:1789763170.043644:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763170.043653:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763170.043705:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507754368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763170.043755:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763170.043760:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763170.043762:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763170.043768:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3147 00000100:00080000:3.0:1789763170.043772:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763170.043776:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763170.043788:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763170.043804:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507754496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763170.043821:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763170.043825:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763170.043828:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763170.043838:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507754368:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763170.043839:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3147 00000100:00080000:3.0:1789763170.043849:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763170.043885:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763170.043893:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763170.043909:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507754624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763170.043922:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763170.043927:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763170.043929:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763170.043934:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3147 00000100:00080000:3.0:1789763170.043940:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763170.043944:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763170.043950:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789763170.043952:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763170.043974:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507754752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763170.044000:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763170.044004:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763170.044006:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763170.044011:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3147 00000100:00080000:3.0:1789763170.044016:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763170.044021:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763170.044027:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763170.044031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffff8d25d1b0b100 x1876699507754368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763170.044047:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507754880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763170.044066:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763170.044072:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763170.044078:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763170.044080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507754368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763170.044087:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3147 00000100:00080000:3.0:1789763170.044094:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763170.044103:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763170.044114:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763170.044134:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763170.044135:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507755008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763170.044157:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763170.044173:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507754368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763170.044217:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507754496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763170.044274:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763170.044295:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffff8d25d1b08380 x1876699507754496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763170.044329:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507754496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763170.044340:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763170.044345:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763170.044350:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507754496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763170.044358:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507754624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763170.044462:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763170.044488:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffff8d25d1b09500 x1876699507754624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763170.044529:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507754624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763170.044542:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763170.044546:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763170.044551:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507754624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763170.044560:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507754752:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763170.044591:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763170.044606:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffff8d25d1b08000 x1876699507754752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763170.044621:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507754752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763170.044631:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763170.044635:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763170.044639:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507754752:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763170.044646:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507754880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763170.044665:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763170.044676:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffff8d25d1b0a300 x1876699507754880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763170.044690:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507754880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763170.044700:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763170.044703:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763170.044707:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507754880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763170.044713:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507755008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763170.044731:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763170.044743:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763170/real 1789763170] req@ffff8d25d1b0b480 x1876699507755008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763170.044757:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507755008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763195 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763170.044775:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763170.044778:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763170.044781:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507755008:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763170.107590:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763171.131604:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763172.155651:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763173.179659:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763174.203640:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763175.163576:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763175.163593:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763175.163602:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763175.163619:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3152 00000100:00080000:2.0:1789763175.163635:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763175.163646:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763175.163715:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507755136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763175.163769:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763175.163778:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763175.163782:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763175.163792:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3152 00000100:00080000:2.0:1789763175.163802:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763175.163847:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763175.163868:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507755136:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763175.163868:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763175.163901:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507755264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763175.163920:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763175.163925:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763175.163929:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763175.163935:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3152 00000100:00080000:2.0:1789763175.163944:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763175.163950:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763175.163957:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763175.163972:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507755392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763175.163997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763175.164003:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763175.164010:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763175.164013:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763175.164020:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3152 00000100:00080000:2.0:1789763175.164024:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763175.164028:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763175.164034:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763175.164062:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507755520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763175.164076:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763175.164081:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763175.164084:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763175.164089:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3152 00000100:00080000:2.0:1789763175.164095:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763175.164101:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763175.164107:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763175.164113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffff8d25c8023100 x1876699507755136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763175.164134:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507755648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763175.164149:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763175.164152:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507755136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763175.164156:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763175.164164:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763175.164181:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3152 00000100:00080000:2.0:1789763175.164191:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763175.164193:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763175.164200:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763175.164209:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789763175.164211:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763175.164220:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507755136:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763175.164232:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507755776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763175.164268:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507755264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763175.164325:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763175.164345:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffff8d25c8022d80 x1876699507755264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763175.164362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507755264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763175.164440:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763175.164445:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763175.164450:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507755264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763175.164459:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507755392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763175.164488:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763175.164504:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffff8d25c8020380 x1876699507755392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763175.164519:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507755392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763175.164530:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763175.164534:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763175.164538:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507755392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763175.164558:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507755520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763175.164582:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763175.164595:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffff8d25c8022300 x1876699507755520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763175.164609:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507755520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763175.164620:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763175.164623:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763175.164629:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507755520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763175.164636:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507755648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763175.164657:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763175.164668:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffff8d25c8023b80 x1876699507755648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763175.164683:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507755648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763175.164693:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763175.164696:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763175.164700:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507755648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763175.164706:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507755776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763175.164726:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763175.164738:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763175/real 1789763175] req@ffff8d25c8022a00 x1876699507755776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763175.164754:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507755776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763200 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763175.164763:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763175.164767:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763175.164770:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507755776:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763175.227744:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763176.251657:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763177.275595:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763178.299641:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763179.323641:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763180.283658:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.283670:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763180.283677:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.283689:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3157 00000100:00080000:0.0:1789763180.283702:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763180.283711:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763180.283944:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507755904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763180.284000:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.284007:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763180.284011:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.284021:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3157 00000100:00080000:0.0:1789763180.284038:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763180.284048:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763180.284063:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507755904:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789763180.284070:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763180.284107:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507756032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763180.284122:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.284127:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763180.284130:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.284137:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3157 00000100:00080000:0.0:1789763180.284143:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763180.284148:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763180.284154:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763180.284170:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507756160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763180.284192:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.284197:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763180.284200:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.284205:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3157 00000100:00080000:0.0:1789763180.284212:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763180.284216:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763180.284221:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763180.284228:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763180.284240:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507756288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763180.284264:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.284270:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763180.284273:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.284279:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3157 00000100:00080000:0.0:1789763180.284286:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763180.284290:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763180.284298:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763180.284311:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507756416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763180.284324:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763180.284331:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763180.284335:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763180.284347:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3157 00000100:00100000:1.0:1789763180.284350:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffff8d25d25ba680 x1876699507755904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763180.284357:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763180.284444:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507755904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763180.284461:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763180.284462:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763180.284481:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789763180.284485:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763180.284493:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507755904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763180.284520:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507756544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763180.284537:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763180.284586:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763180.284612:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffff8d25d25bb800 x1876699507756032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763180.284629:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507756032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763180.284640:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763180.284645:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763180.284649:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763180.284657:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763180.284678:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763180.284690:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffff8d25d25b9f80 x1876699507756160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763180.284704:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507756160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763180.284723:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763180.284727:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763180.284731:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763180.284738:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763180.284758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763180.284769:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffff8d25d25b8380 x1876699507756288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763180.284812:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507756288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763180.284823:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763180.284827:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763180.284831:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763180.284839:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507756416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763180.284866:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763180.284881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffff8d25d25b8700 x1876699507756416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763180.284896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507756416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763180.284905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763180.284909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763180.284913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507756416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763180.284938:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507756544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763180.284959:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763180.284971:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763180/real 1789763180] req@ffff8d25d25bb100 x1876699507756544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763180.284986:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507756544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763205 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763180.284995:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763180.284999:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763180.285003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507756544:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763180.347590:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763181.371659:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763182.395585:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763183.419692:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763184.445363:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763185.403585:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763185.403601:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763185.403608:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763185.403626:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3163 00000100:00080000:1.0:1789763185.403642:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763185.403653:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763185.403719:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507756672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763185.403788:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763185.403797:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763185.403802:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763185.403813:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3163 00000100:00080000:1.0:1789763185.403822:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763185.403833:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763185.403844:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507756672:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789763185.403858:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763185.403892:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507756800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763185.403910:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763185.403916:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763185.403919:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763185.403924:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3163 00000100:00080000:1.0:1789763185.403943:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763185.403949:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789763185.403958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789763185.403958:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763185.403982:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507756928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763185.403997:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763185.404003:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763185.404005:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763185.404011:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3163 00000100:00080000:1.0:1789763185.404016:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763185.404023:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763185.404030:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763185.404039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763185/real 1789763185] req@ffff8d25c8168380 x1876699507756672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763185.404050:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507757056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763185.404081:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763185.404086:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763185.404088:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763185.404092:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3163 00000100:00080000:1.0:1789763185.404097:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763185.404107:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507756672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763185.404109:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763185.404129:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763185.404136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763185.404146:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763185.404153:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507756672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763185.404163:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507757184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763185.404187:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763185.404195:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763185.404200:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756800:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789763185.404218:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763185.404230:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3163 00000100:00080000:1.0:1789763185.404239:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763185.404248:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763185.404253:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763185.404265:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763185.404288:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763185/real 1789763185] req@ffff8d25c8168700 x1876699507756800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763185.404289:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507757312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763185.404362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507756800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763185.404423:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763185.404428:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763185.404434:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763185.404444:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763185.404479:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763185.404497:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763185/real 1789763185] req@ffff8d25c8168a80 x1876699507756928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763185.404514:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507756928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763185.404524:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763185.404528:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763185.404533:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507756928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763185.404540:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763185.404561:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763185.404573:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763185/real 1789763185] req@ffff8d25c816a300 x1876699507757056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763185.404588:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507757056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763185.404598:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763185.404602:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763185.404606:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763185.404612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507757184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763185.404632:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763185.404645:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763185/real 1789763185] req@ffff8d25c816ad80 x1876699507757184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763185.404660:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507757184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763185.404670:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763185.404673:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763185.404677:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507757184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763185.404706:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507757312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763185.404726:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763185.404768:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763185/real 1789763185] req@ffff8d25c8169c00 x1876699507757312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763185.404790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507757312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763210 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763185.404801:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763185.404805:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763185.404809:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507757312:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763185.467662:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763186.491581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763187.515659:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763188.539618:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763189.563608:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763190.523535:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763190.523548:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763190.523553:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763190.523566:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3168 00000100:00080000:3.0:1789763190.523578:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763190.523587:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763190.523635:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507757440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763190.523683:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763190.523688:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763190.523691:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763190.523698:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3168 00000100:00080000:3.0:1789763190.523727:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763190.523734:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763190.523748:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763190.523767:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507757568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763190.523784:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763190.523791:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507757440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763190.523792:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763190.523799:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763190.523813:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3168 00000100:00080000:3.0:1789763190.523817:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763190.523822:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763190.523829:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763190.523844:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507757696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763190.523857:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763190.523862:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763190.523864:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763190.523868:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3168 00000100:00080000:3.0:1789763190.523875:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763190.523879:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763190.523885:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789763190.523894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763190.523908:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507757824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763190.523925:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763190.523929:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763190.523931:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763190.523936:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3168 00000100:00080000:3.0:1789763190.523942:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763190.523947:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763190.523954:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763190.523959:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763190/real 1789763190] req@ffff8d25d1b08a80 x1876699507757440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763190.523973:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507757952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763190.523993:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763190.524000:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763190.524002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507757440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763190.524005:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763190.524021:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3168 00000100:00080000:3.0:1789763190.524031:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763190.524038:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763190.524039:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763190.524054:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763190.524055:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763190.524066:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507757440:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763190.524075:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507758080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763190.524106:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763190.524138:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763190.524155:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763190/real 1789763190] req@ffff8d25d1b0ad80 x1876699507757568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763190.524171:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507757568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763190.524182:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763190.524186:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763190.524190:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763190.524197:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763190.524216:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763190.524228:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763190/real 1789763190] req@ffff8d25d1b09180 x1876699507757696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763190.524242:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507757696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763190.524252:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763190.524256:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763190.524260:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763190.524268:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763190.524286:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763190.524297:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763190/real 1789763190] req@ffff8d25d1b0aa00 x1876699507757824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763190.524311:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507757824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763190.524321:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763190.524324:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763190.524328:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507757824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763190.524334:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507757952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763190.524353:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763190.524370:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763190/real 1789763190] req@ffff8d25d1b09c00 x1876699507757952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763190.524455:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507757952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763190.524471:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763190.524476:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763190.524480:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507757952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763190.524489:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507758080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763190.524531:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763190.524553:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763190/real 1789763190] req@ffff8d25d1b08700 x1876699507758080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763190.524579:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507758080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763215 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763190.524596:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763190.524600:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763190.524605:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507758080:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763190.587696:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763191.611665:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763192.635627:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763193.659594:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763194.683622:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763195.643480:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763195.643492:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763195.643508:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763195.643521:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3173 00000100:00080000:3.0:1789763195.643534:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763195.643543:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763195.643592:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507758208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763195.643620:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763195.643623:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763195.643625:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763195.643628:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3173 00000100:00080000:3.0:1789763195.643631:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763195.643634:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763195.643645:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763195.643680:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507758336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763195.643692:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763195.643697:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763195.643700:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507758208:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763195.643701:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763195.643713:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3173 00000100:00080000:3.0:1789763195.643718:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763195.643723:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763195.643744:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763195.643757:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507758464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763195.643769:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763195.643772:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763195.643774:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763195.643777:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3173 00000100:00080000:3.0:1789763195.643781:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763195.643783:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763195.643787:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763195.643795:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507758592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763195.643802:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763195.643805:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763195.643806:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763195.643809:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3173 00000100:00080000:3.0:1789763195.643811:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763195.643815:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763195.643821:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763195.643830:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507758720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763195.643838:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763195.643840:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507758336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763195.643841:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763195.643845:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763195.643853:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3173 00000100:00080000:3.0:1789763195.643860:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763195.643865:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763195.643870:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763195.643876:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507758464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763195.643889:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507758848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763195.643899:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507758592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763195.643922:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507758720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763195.643942:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507758848:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763195.707538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763196.731574:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763197.755582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763198.715939:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789763198.716004:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763198.716030:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763198.716097:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763198.716119:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763198.716146:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1789763198.716163:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763195/real 1789763198] req@ffff8d25d1b08e00 x1876699507758336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:0.0:1789763198.716170:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763198.716198:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1789763198.716230:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507758336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763198.716257:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763198.716267:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763198.716277:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507758336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763198.716293:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763195/real 1789763198] req@ffff8d25d1b09880 x1876699507758464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763198.716324:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507758464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763198.716342:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763198.716350:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763198.716355:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507758464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763198.716368:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763195/real 1789763198] req@ffff8d25d1b0b800 x1876699507758592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763198.716469:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507758592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763198.716487:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763198.716495:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763198.716504:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507758592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763198.716522:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763195/real 1789763198] req@ffff8d25d1b0bb80 x1876699507758208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763198.716552:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507758208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763198.716569:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763198.716578:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763198.716588:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507758208:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763198.716636:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763195/real 1789763198] req@ffff8d25d1b0a680 x1876699507758720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763198.716690:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507758720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763198.716707:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763198.716714:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763198.716721:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507758720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763198.716734:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763195/real 1789763198] req@ffff8d25d1b08700 x1876699507758848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763198.716758:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507758848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763220 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763198.716775:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763198.716782:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763198.716788:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507758848:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763198.779582:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763199.803698:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763200.763541:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763200.763573:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763200.763581:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763200.763599:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3178 00000100:00080000:3.0:1789763200.763617:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763200.763650:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763200.763717:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507758976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763200.763761:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763200.763767:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763200.763770:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763200.763776:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3178 00000100:00080000:3.0:1789763200.763782:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763200.763788:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763200.763804:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763200.763822:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507759104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763200.763840:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507758976:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763200.763852:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763200.763861:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763200.763863:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763200.763872:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3178 00000100:00080000:3.0:1789763200.763877:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763200.763885:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763200.763894:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763200.763913:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507759232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763200.763931:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763200.763943:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763200.763948:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763200.763956:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763200.763968:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3178 00000100:00080000:3.0:1789763200.763980:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763200.763990:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763200.764015:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763200.764032:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763200/real 1789763200] req@ffff8d25d1b09c00 x1876699507758976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763200.764057:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507759360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763200.764084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507758976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763200.764084:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763200.764113:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763200.764119:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763200.764141:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3178 00000100:00080000:3.0:1789763200.764161:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763200.764182:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763200.764187:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763200.764199:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763200.764206:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763200.764227:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507758976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763200.764247:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507759488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763200.764284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759104:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763200.764290:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763200.764311:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763200.764316:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763200.764328:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3178 00000100:00080000:3.0:1789763200.764341:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763200.764352:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763200.764364:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763200.764371:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763200.764463:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763200/real 1789763200] req@ffff8d25d1b0aa00 x1876699507759104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763200.764486:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507759616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763200.764510:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507759104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763200.764539:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763200.764548:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763200.764556:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763200.764573:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763200.764658:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763200.764692:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763200/real 1789763200] req@ffff8d25d1b09180 x1876699507759232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763200.764721:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507759232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763200.764738:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763200.764745:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763200.764752:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763200.764768:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763200.764818:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763200.764851:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763200/real 1789763200] req@ffff8d25d1b0ad80 x1876699507759360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763200.764878:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507759360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763200.764895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763200.764902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763200.764909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763200.764961:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507759488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763200.765012:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763200.765044:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763200/real 1789763200] req@ffff8d25d1b08a80 x1876699507759488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763200.765071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507759488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763200.765087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763200.765095:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763200.765106:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507759488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763200.765122:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507759616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763200.765173:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763200.765205:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763200/real 1789763200] req@ffff8d25d1b0b480 x1876699507759616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763200.765231:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507759616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763225 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763200.765247:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763200.765255:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763200.765760:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507759616:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763200.827645:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763201.851579:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763202.875665:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763203.899646:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763204.923696:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763205.883649:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763205.883662:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763205.883667:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763205.883679:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3183 00000100:00080000:0.0:1789763205.883690:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763205.883698:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763205.883762:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507759744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763205.883801:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763205.883807:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763205.883809:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763205.883815:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3183 00000100:00080000:0.0:1789763205.883819:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763205.883824:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763205.883838:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763205.883855:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507759872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763205.883870:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507759744:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763205.883875:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763205.883886:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763205.883889:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763205.883895:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3183 00000100:00080000:0.0:1789763205.883899:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763205.883906:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763205.883912:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763205.883928:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507760000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763205.883941:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763205.883945:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763205.883948:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763205.883953:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3183 00000100:00080000:0.0:1789763205.883958:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763205.883962:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789763205.883970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763205.883972:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763205.883991:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507760128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763205.884004:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763205.884009:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763205.884011:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763205.884018:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3183 00000100:00080000:0.0:1789763205.884023:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763205.884034:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763205.884036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763205/real 1789763205] req@ffff8d25d25b9c00 x1876699507759744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763205.884041:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763205.884063:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507760256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763205.884078:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763205.884086:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763205.884088:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507759744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763205.884089:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763205.884104:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3183 00000100:00080000:0.0:1789763205.884129:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763205.884140:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763205.884153:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763205.884157:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763205.884168:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507760384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763205.884175:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763205.884191:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507759744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763205.884249:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763205.884286:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763205.884304:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763205/real 1789763205] req@ffff8d25d25b8e00 x1876699507759872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763205.884320:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507759872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763205.884340:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763205.884362:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763205.884367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507759872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763205.884374:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763205.884395:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763205.884406:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763205/real 1789763205] req@ffff8d25d25bb480 x1876699507760000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763205.884420:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507760000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763205.884478:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763205.884483:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763205.884487:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763205.884495:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763205.884522:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763205.884537:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763205/real 1789763205] req@ffff8d25d25bb100 x1876699507760128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763205.884552:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507760128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763205.884562:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763205.884566:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763205.884571:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763205.884578:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507760256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763205.884651:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763205.884666:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763205/real 1789763205] req@ffff8d25d25b8700 x1876699507760256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763205.884679:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507760256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763205.884689:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763205.884692:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763205.884696:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507760256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763205.884703:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507760384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763205.884722:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763205.884733:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763205/real 1789763205] req@ffff8d25d25b8380 x1876699507760384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763205.884746:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507760384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763230 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763205.884754:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763205.884757:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763205.884760:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507760384:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763205.947666:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763206.971565:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763207.995565:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763209.019658:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763210.043593:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763211.003601:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763211.003614:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763211.003620:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763211.003633:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3188 00000100:00080000:3.0:1789763211.003644:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763211.003653:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763211.003699:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507760512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763211.003738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763211.003744:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763211.003747:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763211.003752:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3188 00000100:00080000:3.0:1789763211.003757:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763211.003762:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763211.003774:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763211.003790:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507760640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763211.003808:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763211.003811:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507760512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763211.003813:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763211.003821:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763211.003836:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3188 00000100:00080000:3.0:1789763211.003840:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763211.003844:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763211.003849:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763211.003867:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507760768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763211.003879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763211.003884:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763211.003887:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763211.003892:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3188 00000100:00080000:3.0:1789763211.003897:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763211.003901:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763211.003909:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763211.003922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763211.003927:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507760896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763211.003959:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763211.003964:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763211.003967:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763211.003974:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3188 00000100:00080000:3.0:1789763211.003980:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763211.003986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffff8d25d1b0a300 x1876699507760512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763211.003992:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763211.004007:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763211.004027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507760512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763211.004038:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507761024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763211.004049:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763211.004068:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763211.004079:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763211.004083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507760512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763211.004092:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763211.004101:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763211.004127:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3188 00000100:00100000:1.0:1789763211.004137:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763211.004139:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763211.004157:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763211.004167:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763211.004191:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507761152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763211.004207:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763211.004248:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffff8d25d1b08000 x1876699507760640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763211.004278:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507760640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763211.004295:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763211.004300:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763211.004304:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763211.004316:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763211.004359:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763211.004429:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffff8d25d1b09500 x1876699507760768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763211.004449:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507760768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763211.004468:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763211.004472:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763211.004476:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763211.004484:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763211.004514:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763211.004528:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffff8d25d1b08380 x1876699507760896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763211.004542:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507760896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763211.004585:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763211.004589:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763211.004594:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507760896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763211.004602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763211.004631:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763211.004646:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffff8d25d1b0b100 x1876699507761024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763211.004660:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507761024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763211.004670:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763211.004674:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763211.004679:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763211.004708:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763211.004729:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763211.004742:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763211/real 1789763211] req@ffff8d25d1b0a300 x1876699507761152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763211.004756:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507761152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763236 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763211.004772:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763211.004779:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763211.004786:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761152:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763211.067547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763212.091647:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763213.115629:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763214.139645:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763215.163602:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763216.123550:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763216.123567:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763216.123575:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763216.123591:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3193 00000100:00080000:0.0:1789763216.123605:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763216.123617:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763216.123672:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507761280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763216.123712:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763216.123718:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763216.123720:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763216.123726:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3193 00000100:00080000:0.0:1789763216.123732:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763216.123738:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763216.123751:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763216.123772:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507761280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763216.123775:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507761408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763216.123798:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763216.123804:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763216.123806:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763216.123811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3193 00000100:00080000:0.0:1789763216.123816:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763216.123821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763216.123827:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763216.123842:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507761536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763216.123854:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763216.123859:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763216.123861:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763216.123865:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3193 00000100:00080000:0.0:1789763216.123870:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763216.123881:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789763216.123881:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763216.123896:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763216.123910:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507761664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763216.123924:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763216.123928:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763216.123930:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763216.123935:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3193 00000100:00080000:0.0:1789763216.123939:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763216.123944:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763216.123952:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763216.123964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763216] req@ffff8d25d25b9f80 x1876699507761280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763216.123968:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507761792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763216.124027:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763216.124033:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763216.124036:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763216.124043:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3193 00000100:00080000:0.0:1789763216.124047:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763216.124052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507761280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763216.124057:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763216.124069:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763216.124087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763216.124092:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507761920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763216.124099:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763216.124115:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507761280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763216.124156:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507761408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763216.124193:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763216.124212:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763216] req@ffff8d25d25bb800 x1876699507761408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763216.124229:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507761408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763216.124240:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763216.124244:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763216.124249:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507761408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763216.124256:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507761536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763216.124276:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763216.124288:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763216] req@ffff8d25d25ba680 x1876699507761536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763216.124302:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507761536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763216.124311:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763216.124315:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763216.124318:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507761536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763216.124325:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507761664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763216.124343:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763216.124353:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763216] req@ffff8d25d25bbb80 x1876699507761664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763216.124367:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507761664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763216.124410:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763216.124414:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763216.124418:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507761664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763216.124426:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763216.124450:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763216.124463:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763216] req@ffff8d25d25b9180 x1876699507761792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763216.124478:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507761792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763216.124488:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763216.124492:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763216.124496:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763216.124502:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763216.124550:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763216.124564:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763216/real 1789763216] req@ffff8d25d25b8a80 x1876699507761920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763216.124579:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507761920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763241 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763216.124589:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763216.124592:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763216.124597:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507761920:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763216.187586:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763217.211630:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763218.235646:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763219.259646:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763220.283667:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763221.243518:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763221.243528:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763221.243533:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763221.243544:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3198 00000100:00080000:3.0:1789763221.243554:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763221.243561:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763221.243597:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507762048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763221.243625:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763221.243629:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763221.243631:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763221.243636:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3198 00000100:00080000:3.0:1789763221.243639:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763221.243644:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763221.243655:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763221.243667:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507762176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763221.243674:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507762048:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763221.243680:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763221.243687:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763221.243689:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763221.243693:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3198 00000100:00080000:3.0:1789763221.243696:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763221.243699:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763221.243705:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763221.243715:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507762304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763221.243724:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763221.243728:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763221.243730:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763221.243733:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3198 00000100:00080000:3.0:1789763221.243737:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763221.243740:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763221.243744:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763221.243754:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507762432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763221.243760:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763221.243782:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763221.243786:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763221.243787:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763221.243792:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3198 00000100:00080000:3.0:1789763221.243796:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763221.243800:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763221.243806:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763221.243811:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffff8d25d1b0b480 x1876699507762048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763221.243822:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507762560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763221.243835:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763221.243840:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507762048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763221.243842:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763221.243847:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763221.243857:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3198 00000100:00080000:3.0:1789763221.243864:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763221.243865:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763221.243870:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763221.243876:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789763221.243879:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763221.243884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507762048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763221.243892:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507762688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763221.243931:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763221.243957:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763221.243970:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffff8d25d1b08a80 x1876699507762176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763221.243981:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507762176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763221.243989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763221.243992:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763221.243995:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763221.244000:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763221.244014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763221.244023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffff8d25d1b0ad80 x1876699507762304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763221.244033:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507762304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763221.244040:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763221.244043:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763221.244045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763221.244050:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762432:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763221.244064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763221.244072:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffff8d25d1b09180 x1876699507762432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763221.244082:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507762432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763221.244089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763221.244092:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763221.244094:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763221.244098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507762560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763221.244112:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763221.244120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffff8d25d1b0aa00 x1876699507762560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763221.244130:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507762560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763221.244137:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763221.244140:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763221.244143:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507762560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763221.244148:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507762688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763221.244160:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763221.244168:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763221/real 1789763221] req@ffff8d25d1b09c00 x1876699507762688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763221.244177:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507762688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763246 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763221.244184:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763221.244186:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763221.244189:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507762688:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763221.307632:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763222.331503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763223.355623:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763224.379615:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763225.403656:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763226.363615:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.363627:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763226.363633:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763226.363645:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3204 00000100:00080000:0.0:1789763226.363658:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.363667:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763226.363714:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507762816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763226.363752:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.363757:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763226.363760:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763226.363765:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3204 00000100:00080000:0.0:1789763226.363771:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.363778:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763226.363790:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763226.363813:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507762816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763226.363814:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507762944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763226.363842:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.363847:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763226.363849:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763226.363854:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3204 00000100:00080000:0.0:1789763226.363857:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.363861:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763226.363868:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763226.363885:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507763072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763226.363899:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.363903:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763226.363905:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763226.363910:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3204 00000100:00080000:0.0:1789763226.363916:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.363920:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789763226.363940:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763226.363944:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763226.363966:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507763200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763226.363989:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.363994:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763226.363997:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763226.364003:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3204 00000100:00080000:0.0:1789763226.364008:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.364018:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763226.364022:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffff8d25d25b8000 x1876699507762816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763226.364025:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763226.364048:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507763328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763226.364071:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763226.364076:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763226.364078:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763226.364087:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507762816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763226.364089:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3204 00000100:00080000:0.0:1789763226.364097:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763226.364107:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763226.364121:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763226.364136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763226.364145:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507763456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763226.364151:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763226.364167:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507762816:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763226.364215:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763226.364253:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763226.364274:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffff8d25d25ba300 x1876699507762944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763226.364293:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507762944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763226.364306:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763226.364312:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763226.364316:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507762944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763226.364324:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763226.364344:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763226.364359:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffff8d25d25bad80 x1876699507763072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763226.364408:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507763072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763226.364419:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763226.364423:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763226.364428:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763226.364436:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763226.364508:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763226.364534:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffff8d25d25b8a80 x1876699507763200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763226.364560:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507763200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763226.364578:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763226.364585:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763226.364591:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763226.364599:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507763328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763226.364627:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763226.364642:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffff8d25d25b9180 x1876699507763328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763226.364657:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507763328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763226.364667:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763226.364671:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763226.364674:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507763328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763226.364681:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507763456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763226.364699:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763226.364711:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763226/real 1789763226] req@ffff8d25d25bbb80 x1876699507763456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763226.364724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507763456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763251 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763226.364735:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763226.364739:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763226.364742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507763456:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763226.427641:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763227.451624:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763228.475569:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763229.499674:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763230.523689:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763231.483656:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763231.483670:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763231.483676:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763231.483690:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3209 00000100:00080000:1.0:1789763231.483722:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763231.483730:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763231.483772:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507763584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763231.483803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763231.483807:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763231.483809:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763231.483813:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3209 00000100:00080000:1.0:1789763231.483817:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763231.483823:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763231.483834:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763231.483857:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507763712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763231.483875:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763231.483877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507763584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763231.483884:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763231.483894:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763231.483902:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3209 00000100:00080000:1.0:1789763231.483906:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763231.483909:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763231.483915:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763231.483933:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507763840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763231.483943:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763231.483947:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763231.483948:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763231.483953:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3209 00000100:00080000:1.0:1789763231.483956:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763231.483960:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763231.483966:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763231.483978:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507763968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763231.483988:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763231.483991:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763231.483993:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763231.483996:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3209 00000100:00080000:1.0:1789763231.484000:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763231.484003:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763231.484008:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763231.484027:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507764096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763231.484028:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763231.484046:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763231.484051:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763231.484055:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763231.484059:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3209 00000100:00080000:1.0:1789763231.484063:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763231.484067:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763231.484074:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763231.484104:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507764224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763231.484133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763231/real 1789763231] req@ffff8d25c8169880 x1876699507763584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763231.484170:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507763584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763231.484186:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763231.484193:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763231.484198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507763584:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763231.484235:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763231.484264:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763231.484278:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763231/real 1789763231] req@ffff8d25c816b480 x1876699507763712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763231.484292:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507763712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763231.484301:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763231.484305:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763231.484308:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763231.484314:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763231.484331:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763231.484341:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763231/real 1789763231] req@ffff8d25c8169c00 x1876699507763840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763231.484354:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507763840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763231.484369:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763231.484372:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763231.484446:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763231.484456:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763231.484500:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763231.484515:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763231/real 1789763231] req@ffff8d25c816ad80 x1876699507763968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763231.484531:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507763968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763231.484542:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763231.484547:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763231.484550:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507763968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763231.484557:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763231.484577:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763231.484593:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763231/real 1789763231] req@ffff8d25c816a300 x1876699507764096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763231.484608:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507764096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763231.484617:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763231.484621:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763231.484626:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763231.484633:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763231.484653:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763231.484666:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763231/real 1789763231] req@ffff8d25c8168a80 x1876699507764224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763231.484680:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507764224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763256 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763231.484690:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763231.484693:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763231.484697:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764224:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763231.547625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763232.571623:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763233.595628:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763234.619650:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763235.643658:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763236.603650:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763236.603664:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763236.603671:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763236.603702:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3214 00000100:00080000:3.0:1789763236.603719:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763236.603731:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763236.603792:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507764352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763236.603837:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763236.603844:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763236.603847:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763236.603854:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3214 00000100:00080000:3.0:1789763236.603859:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763236.603865:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763236.603886:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763236.603922:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507764480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763236.603934:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507764352:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763236.603985:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763236.603994:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763236.604000:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763236.604058:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763236.604138:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763236/real 1789763236] req@ffff8d25d1b08700 x1876699507764352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763236.604173:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507764352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763236.604194:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763236.604201:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763236.604207:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507764352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763236.604241:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507764480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763236.604277:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763236.604300:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763236/real 1789763236] req@ffff8d25d1b0a680 x1876699507764480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763236.604320:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507764480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763236.604332:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763236.604335:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3214 00000100:00080000:1.0:1789763236.604343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763236.604357:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763236.604358:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507764480:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763236.604457:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763236.604472:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763236.604502:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507764608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763236.604533:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763236.604540:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763236.604543:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763236.604553:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3214 00000100:00080000:3.0:1789763236.604564:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763236.604571:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763236.604582:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507764608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763236.604625:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763236.604646:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763236.604652:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763236/real 1789763236] req@ffff8d25d1b0bb80 x1876699507764608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763236.604672:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507764736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763236.604685:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507764608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763236.604696:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763236.604717:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763236.604723:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763236.604737:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3214 00000100:00080000:1.0:1789763236.604743:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763236.604744:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763236.604761:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763236.604772:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789763236.604776:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763236.604790:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507764608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763236.604799:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507764864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763236.604821:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763236.604826:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763236.604829:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763236.604836:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507764736:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789763236.604837:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3214 00000100:00080000:3.0:1789763236.604855:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763236.604862:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763236.604870:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763236.604894:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507764992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763236.604897:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763236.604923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763236/real 1789763236] req@ffff8d25d1b0b800 x1876699507764736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763236.604951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507764736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763236.604965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763236.604973:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763236.604979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507764736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763236.604988:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763236.605010:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763236.605023:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763236/real 1789763236] req@ffff8d25d1b09880 x1876699507764864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763236.605037:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507764864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763236.605047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763236.605050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763236.605054:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763236.605073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763236.605093:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763236.605105:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763236/real 1789763236] req@ffff8d25d1b08e00 x1876699507764992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763236.605120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507764992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763261 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763236.605129:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763236.605133:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763236.605138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507764992:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763236.667645:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763237.691648:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763238.715627:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763239.739608:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763240.763615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763241.723605:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763241.723622:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763241.723629:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763241.723656:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3219 00000100:00080000:0.0:1789763241.723671:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763241.723684:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763241.723744:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507765120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763241.723796:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763241.723802:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763241.723805:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763241.723810:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3219 00000100:00080000:0.0:1789763241.723816:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763241.723824:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763241.723838:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763241.723867:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507765248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763241.723867:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507765120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763241.723898:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763241.723902:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763241.723905:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763241.723910:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3219 00000100:00080000:0.0:1789763241.723915:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763241.723919:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763241.723925:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763241.723941:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507765376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763241.723953:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763241.723957:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763241.723960:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763241.723964:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3219 00000100:00080000:0.0:1789763241.723985:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763241.723989:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763241.723997:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763241.724007:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763241.724031:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507765504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763241.724048:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763241.724053:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763241.724054:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763241.724059:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3219 00000100:00080000:0.0:1789763241.724063:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763241.724069:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763241.724075:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763241.724097:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763241/real 1789763241] req@ffff8d25d25ba680 x1876699507765120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763241.724104:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507765632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763241.724153:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763241.724157:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763241.724159:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763241.724167:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507765120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763241.724171:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3219 00000100:00080000:0.0:1789763241.724182:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763241.724193:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763241.724203:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763241.724204:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763241.724216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763241.724223:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507765760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763241.724231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507765120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763241.724281:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507765248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763241.724318:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763241.724371:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763241/real 1789763241] req@ffff8d25d25bb800 x1876699507765248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763241.724423:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507765248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763241.724434:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763241.724438:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763241.724443:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507765248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763241.724451:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507765376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763241.724480:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763241.724495:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763241/real 1789763241] req@ffff8d25d25b9f80 x1876699507765376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763241.724510:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507765376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763241.724520:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763241.724523:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763241.724527:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507765376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763241.724534:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507765504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763241.724553:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763241.724565:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763241/real 1789763241] req@ffff8d25d25b8380 x1876699507765504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763241.724579:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507765504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763241.724599:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763241.724602:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763241.724606:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507765504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763241.724612:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507765632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763241.724630:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763241.724641:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763241/real 1789763241] req@ffff8d25d25b8700 x1876699507765632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763241.724655:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507765632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763241.724666:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763241.724669:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763241.724673:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507765632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763241.724679:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507765760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763241.724697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763241.724708:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763241/real 1789763241] req@ffff8d25d25bb100 x1876699507765760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763241.724724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507765760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763266 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763241.724740:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763241.724746:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763241.724752:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507765760:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763241.787580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763242.811589:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763243.835595:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763244.859589:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763245.883655:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763246.843546:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763246.843564:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763246.843571:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763246.843588:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3224 00000100:00080000:3.0:1789763246.843603:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763246.843623:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763246.843689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507765888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763246.843738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763246.843743:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763246.843746:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763246.843753:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3224 00000100:00080000:3.0:1789763246.843758:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763246.843765:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763246.843781:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763246.843803:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507765888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763246.843805:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507766016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763246.843833:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763246.843837:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763246.843840:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763246.843847:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3224 00000100:00080000:3.0:1789763246.843851:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763246.843856:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763246.843863:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763246.843881:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507766144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763246.843903:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763246.843908:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763246.843912:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763246.843917:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3224 00000100:00080000:3.0:1789763246.843923:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763246.843932:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789763246.843934:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763246.843945:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763246.843967:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507766272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763246.843981:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763246.843986:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763246.843988:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763246.843993:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3224 00000100:00080000:3.0:1789763246.843998:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763246.844002:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763246.844008:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763246.844042:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763246/real 1789763246] req@ffff8d25d1b08e00 x1876699507765888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763246.844043:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507766400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763246.844081:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763246.844094:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763246.844100:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763246.844108:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3224 00000100:00080000:3.0:1789763246.844114:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763246.844120:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763246.844128:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507765888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763246.844131:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763246.844155:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507766528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763246.844156:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763246.844174:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763246.844190:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507765888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763246.844249:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763246.844282:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763246.844342:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763246/real 1789763246] req@ffff8d25d1b09880 x1876699507766016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763246.844370:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507766016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763246.844425:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763246.844430:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763246.844436:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763246.844445:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763246.844478:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763246.844495:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763246/real 1789763246] req@ffff8d25d1b0b800 x1876699507766144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763246.844511:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507766144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763246.844521:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763246.844525:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763246.844529:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763246.844536:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763246.844557:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763246.844571:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763246/real 1789763246] req@ffff8d25d1b0bb80 x1876699507766272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763246.844585:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507766272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763246.844596:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763246.844599:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763246.844603:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763246.844609:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507766400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763246.844630:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763246.844642:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763246/real 1789763246] req@ffff8d25d1b0a680 x1876699507766400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763246.844657:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507766400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763246.844666:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763246.844670:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763246.844673:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507766400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763246.844679:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507766528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763246.844698:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763246.844709:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763246/real 1789763246] req@ffff8d25d1b08700 x1876699507766528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763246.844723:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507766528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763271 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763246.844733:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763246.844737:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763246.844741:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507766528:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763246.907600:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763247.931662:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763248.955639:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763249.979599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763251.003600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763251.963543:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763251.963557:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763251.963562:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763251.963575:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3229 00000100:00080000:2.0:1789763251.963587:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763251.963596:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763251.963643:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507766656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763251.963692:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763251.963698:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763251.963701:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763251.963708:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3229 00000100:00080000:2.0:1789763251.963713:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763251.963720:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763251.963734:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763251.963747:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507766656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763251.963767:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507766784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763251.963795:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763251.963802:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763251.963807:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763251.963815:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3229 00000100:00080000:2.0:1789763251.963824:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763251.963833:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789763251.963844:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763251.963851:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763251.963884:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507766912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763251.963905:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763251/real 1789763251] req@ffff8d25c8020000 x1876699507766656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763251.963911:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763251.963929:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763251.963936:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763251.963949:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3229 00000100:00100000:1.0:1789763251.963958:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507766656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763251.963961:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763251.963976:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763251.963991:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:1.0:1789763251.964000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763251.964016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763251.964018:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507767040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763251.964037:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507766656:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763251.964042:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763251.964057:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763251.964061:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763251.964068:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3229 00000100:00080000:2.0:1789763251.964072:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763251.964077:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763251.964082:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763251.964093:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507767168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763251.964108:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763251.964115:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763251.964117:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766784:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763251.964120:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763251.964135:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3229 00000100:00080000:2.0:1789763251.964141:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763251.964146:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763251.964154:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763251.964170:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507767296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763251.964183:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763251.964219:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763251/real 1789763251] req@ffff8d25c8020a80 x1876699507766784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763251.964238:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507766784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763251.964276:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763251.964282:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763251.964287:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763251.964297:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763251.964326:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763251.964342:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763251/real 1789763251] req@ffff8d25c8021180 x1876699507766912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763251.964357:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507766912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763251.964367:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763251.964372:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763251.964408:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507766912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763251.964417:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763251.964442:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763251.964457:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763251/real 1789763251] req@ffff8d25c8022680 x1876699507767040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763251.964471:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507767040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763251.964482:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763251.964486:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763251.964490:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763251.964509:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507767168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763251.964530:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763251.964542:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763251/real 1789763251] req@ffff8d25c8023800 x1876699507767168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763251.964558:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507767168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763251.964567:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763251.964585:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763251.964589:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507767168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763251.964597:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507767296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763251.964618:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763251.964631:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763251/real 1789763251] req@ffff8d25c8020000 x1876699507767296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763251.964646:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507767296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763276 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763251.964655:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763251.964659:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763251.964662:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507767296:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763252.027576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763253.051611:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763254.075580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763255.099602:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763256.123646:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763257.083576:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.083591:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763257.083599:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.083615:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3234 00000100:00080000:2.0:1789763257.083632:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.083643:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763257.083707:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507767424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763257.083750:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.083755:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763257.083758:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.083765:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3234 00000100:00080000:2.0:1789763257.083770:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.083776:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763257.083792:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763257.083818:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507767552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763257.083820:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507767424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763257.083848:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.083853:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763257.083855:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.083860:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3234 00000100:00080000:2.0:1789763257.083863:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.083868:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763257.083875:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763257.083889:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507767680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763257.083902:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.083906:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763257.083908:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.083913:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3234 00000100:00080000:2.0:1789763257.083918:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.083922:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763257.083930:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763257.083944:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507767808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763257.083952:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763257.083970:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.083975:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763257.083978:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.083986:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3234 00000100:00080000:2.0:1789763257.083992:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.083996:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763257.084002:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763257.084016:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507767936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763257.084028:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763257.084033:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763257.084036:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763257.084045:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3234 00000100:00100000:1.0:1789763257.084047:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffff8d25c8022a00 x1876699507767424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763257.084050:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763257.084063:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763257.084079:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763257.084101:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507767424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763257.084104:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507768064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763257.084172:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763257.084181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763257.084190:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507767424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763257.084263:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763257.084311:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763257.084332:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffff8d25c8023b80 x1876699507767552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763257.084349:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507767552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763257.084360:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763257.084365:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763257.084369:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763257.084409:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763257.084440:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763257.084455:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffff8d25c8022300 x1876699507767680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763257.084470:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507767680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763257.084481:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763257.084485:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763257.084489:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763257.084497:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767808:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763257.084519:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763257.084530:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffff8d25c8020380 x1876699507767808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763257.084545:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507767808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763257.084555:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763257.084559:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763257.084562:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507767808:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763257.084568:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507767936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763257.084588:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763257.084599:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffff8d25c8022d80 x1876699507767936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763257.084614:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507767936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763257.084624:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763257.084627:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763257.084630:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507767936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763257.084636:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507768064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763257.084655:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763257.084667:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763257/real 1789763257] req@ffff8d25c8023100 x1876699507768064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763257.084682:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507768064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763282 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763257.084692:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763257.084695:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763257.084699:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507768064:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763257.147590:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763258.171659:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763259.195640:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763260.219644:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763261.243624:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763262.203586:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763262.203597:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763262.203603:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763262.203617:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3239 00000100:00080000:0.0:1789763262.203629:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763262.203639:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763262.203694:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507768192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763262.203744:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763262.203752:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763262.203756:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763262.203762:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3239 00000100:00080000:0.0:1789763262.203768:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763262.203775:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763262.203788:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763262.203812:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507768320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763262.203831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507768192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763262.203840:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763262.203853:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763262.203857:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763262.203867:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3239 00000100:00080000:0.0:1789763262.203876:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763262.203884:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763262.203896:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763262.203921:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507768448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763262.203936:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763262.203941:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763262.203944:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763262.203949:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3239 00000100:00080000:0.0:1789763262.203955:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763262.203961:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763262.203968:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763262.203986:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507768576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763262.204001:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763262.204005:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763262.204008:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763262.204021:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3239 00000100:00100000:1.0:1789763262.204022:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507768320:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763262.204026:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763262.204040:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763262.204049:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763262.204072:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507768704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763262.204088:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763262.204095:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763262.204099:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763262.204107:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507768448:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763262.204114:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3239 00000100:00080000:0.0:1789763262.204125:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763262.204132:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763262.204139:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763262.204163:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507768832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763262.204204:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507768576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763262.204274:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507768704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763262.204302:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507768832:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763262.267604:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763263.291608:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763264.315634:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:3.0:1789763265.275882:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:3.0:1789763265.275970:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:3.0:1789763265.276000:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763265.276106:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763265.276137:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763265.276176:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763265.276194:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:3.0:1789763265.276207:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1789763265.276455:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763265] req@ffff8d25d25b8700 x1876699507768704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763265.276492:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507768704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763265.276511:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763265.276520:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763265.276529:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507768704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763265.276548:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763265] req@ffff8d25d25b8380 x1876699507768832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763265.276575:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507768832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763265.276591:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763265.276598:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763265.276606:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507768832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763265.276629:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763265] req@ffff8d25d25b8e00 x1876699507768320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763265.276654:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507768320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763265.276670:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763265.276676:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763265.276683:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507768320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763265.276694:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763265] req@ffff8d25d25b9c00 x1876699507768448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763265.276718:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507768448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763265.276733:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763265.276741:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763265.276748:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507768448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763265.276761:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763265] req@ffff8d25d25bb100 x1876699507768576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763265.276785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507768576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763265.276801:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763265.276808:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763265.276815:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507768576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763265.276827:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763262/real 1789763265] req@ffff8d25d25bb480 x1876699507768192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763265.276849:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507768192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763287 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763265.276865:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763265.276871:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763265.276878:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507768192:192.168.203.141@tcp:250:kworker.0 00010000:00010000:2.0:1789763265.339616:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763266.363626:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763267.323518:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763267.323531:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763267.323537:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763267.323562:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3245 00000100:00080000:3.0:1789763267.323574:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763267.323583:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763267.323627:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507768960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763267.323661:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763267.323666:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763267.323669:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763267.323674:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3245 00000100:00080000:3.0:1789763267.323678:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763267.323696:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763267.323719:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763267.323745:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507769088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763267.323772:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763267.323775:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507768960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763267.323787:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763267.323796:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763267.323807:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3245 00000100:00080000:3.0:1789763267.323815:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763267.323821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763267.323831:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763267.323861:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507769216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763267.323881:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763267.323888:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763267.323897:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763267.323903:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763267.323913:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3245 00000100:00080000:3.0:1789763267.323923:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763267.323932:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763267.323943:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763267.323949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffff8d25d1b09c00 x1876699507768960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763267.323971:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507769344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763267.323994:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763267.323998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507768960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763267.324005:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763267.324013:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763267.324032:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3245 00000100:00080000:1.0:1789763267.324043:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763267.324045:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763267.324062:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763267.324064:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789763267.324080:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763267.324081:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507768960:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763267.324116:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507769472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763267.324164:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763267.324166:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769088:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763267.324177:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763267.324185:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763267.324195:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3245 00000100:00080000:3.0:1789763267.324204:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763267.324212:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763267.324223:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763267.324236:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763267.324255:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507769600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763267.324274:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffff8d25d1b0aa00 x1876699507769088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763267.324304:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507769088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763267.324322:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763267.324329:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763267.324337:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763267.324351:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763267.324444:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763267.324468:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffff8d25d1b09180 x1876699507769216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763267.324491:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507769216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763267.324505:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763267.324511:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763267.324517:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763267.324530:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763267.324570:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763267.324594:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffff8d25d1b0ad80 x1876699507769344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763267.324617:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507769344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763267.324633:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763267.324640:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763267.324646:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763267.324686:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507769472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763267.324741:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763267.324766:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffff8d25d1b08a80 x1876699507769472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763267.324785:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507769472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763267.324800:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763267.324807:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763267.324814:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507769472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763267.324826:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507769600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763267.324867:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763267.324891:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763267/real 1789763267] req@ffff8d25d1b0b480 x1876699507769600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763267.324926:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507769600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763292 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763267.324937:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763267.324941:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763267.324945:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507769600:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763267.387568:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763268.411579:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763269.435662:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763270.459631:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763271.483598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763272.443522:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763272.443534:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763272.443541:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763272.443556:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3250 00000100:00080000:3.0:1789763272.443569:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763272.443581:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763272.443628:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507769728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763272.443667:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763272.443675:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763272.443678:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763272.443684:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3250 00000100:00080000:3.0:1789763272.443689:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763272.443694:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763272.443706:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763272.443733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507769856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763272.443736:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507769728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763272.443770:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763272.443776:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763272.443780:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763272.443786:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3250 00000100:00080000:3.0:1789763272.443792:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763272.443798:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763272.443805:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763272.443828:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507769984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763272.443849:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763272.443855:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763272.443865:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763272.443870:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763272.443882:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3250 00000100:00080000:3.0:1789763272.443893:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763272.443902:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763272.443909:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763272/real 1789763272] req@ffff8d25d1b0a300 x1876699507769728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763272.443919:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763272.443948:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507769728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763272.443954:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507770112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763272.443983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763272.443990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763272.443996:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507769728:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763272.444031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763272.444087:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763272.444149:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763272/real 1789763272] req@ffff8d25d1b0b100 x1876699507769856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763272.444168:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507769856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763272.444190:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763272.444194:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763272.444200:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763272.444209:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763272.444241:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763272.444257:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763272/real 1789763272] req@ffff8d25d1b08380 x1876699507769984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763272.444275:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507769984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763272.444286:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763272.444290:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763272.444297:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507769984:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763272.444311:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763272.444322:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:1.0:1789763272.444323:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770112:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789763272.444332:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763272.444345:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3250 00000100:00080000:3.0:1789763272.444354:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763272.444366:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763272.444367:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763272.444439:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763272/real 1789763272] req@ffff8d25d1b09500 x1876699507770112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763272.444462:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507770112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763272.444477:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763272.444487:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789763272.444489:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763272.444500:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763272.444523:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507770240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763272.444552:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763272.444557:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763272.444560:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763272.444567:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3250 00000100:00080000:3.0:1789763272.444573:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763272.444580:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507770240:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763272.444584:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763272.444602:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763272.444620:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507770368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763272.444651:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763272.444683:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763272/real 1789763272] req@ffff8d25d1b08000 x1876699507770240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763272.444708:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507770240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763272.444724:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763272.444731:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763272.444738:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507770240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763272.444765:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507770368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763272.444808:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763272.444827:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763272/real 1789763272] req@ffff8d25d1b09500 x1876699507770368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763272.444843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507770368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763297 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763272.444853:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763272.444857:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763272.444861:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507770368:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763272.507586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763273.531558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763274.555604:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763275.579693:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763276.603599:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763277.563446:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763277.563456:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763277.563461:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763277.563472:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3255 00000100:00080000:3.0:1789763277.563481:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763277.563489:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763277.563532:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507770496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763277.563563:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763277.563567:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763277.563569:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763277.563573:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3255 00000100:00080000:3.0:1789763277.563577:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763277.563582:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763277.563592:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763277.563600:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507770496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763277.563604:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507770624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763277.563614:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763277.563617:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763277.563619:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763277.563622:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3255 00000100:00080000:3.0:1789763277.563625:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763277.563628:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763277.563632:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763277.563641:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507770752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763277.563655:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763277.563657:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763277.563658:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763277.563660:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763277.563663:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3255 00000100:00080000:3.0:1789763277.563666:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763277.563669:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763277.563673:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763277.563683:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507770880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763277.563692:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763277.563694:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763277/real 1789763277] req@ffff8d25d1b08380 x1876699507770496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763277.563695:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763277.563697:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763277.563701:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3255 00000100:00080000:3.0:1789763277.563704:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763277.563708:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763277.563711:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507770496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763277.563712:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763277.563721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507771008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763277.563722:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763277.563726:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763277.563730:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507770496:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763277.563731:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763277.563734:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763277.563736:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763277.563739:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3255 00000100:00080000:3.0:1789763277.563742:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763277.563746:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763277.563749:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763277.563755:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763277.563758:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507771136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763277.563776:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763277.563789:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763277/real 1789763277] req@ffff8d25d1b0b100 x1876699507770624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763277.563803:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507770624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763277.563810:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763277.563812:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763277.563815:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763277.563819:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763277.563835:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763277.563843:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763277/real 1789763277] req@ffff8d25d1b0a300 x1876699507770752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763277.563850:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507770752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763277.563860:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763277.563862:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763277.563864:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763277.563867:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763277.563878:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763277.563883:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763277/real 1789763277] req@ffff8d25d1b0b480 x1876699507770880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763277.563891:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507770880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763277.563896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763277.563898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763277.563900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507770880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763277.563903:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763277.563912:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763277.563918:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763277/real 1789763277] req@ffff8d25d1b08a80 x1876699507771008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763277.563926:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507771008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763277.563930:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763277.563932:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763277.563934:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763277.563958:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763277.563969:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763277.563975:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763277/real 1789763277] req@ffff8d25d1b0ad80 x1876699507771136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763277.563983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507771136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763302 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763277.563988:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763277.563989:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763277.563991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771136:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763277.627494:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763278.651596:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763279.675551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763280.699530:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763281.723624:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763282.683521:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763282.683536:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763282.683541:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763282.683554:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3260 00000100:00080000:0.0:1789763282.683566:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763282.683574:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763282.683624:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507771264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763282.683649:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763282.683652:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763282.683654:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763282.683658:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3260 00000100:00080000:0.0:1789763282.683662:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763282.683666:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763282.683676:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763282.683687:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507771264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763282.683689:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507771392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763282.683702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763282.683705:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763282.683706:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763282.683709:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3260 00000100:00080000:0.0:1789763282.683711:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763282.683714:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763282.683718:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763282.683726:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507771520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763282.683733:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763282.683736:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763282.683737:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763282.683739:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3260 00000100:00080000:0.0:1789763282.683757:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763282.683759:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763282.683762:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763282.683770:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763282.683773:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507771648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763282.683782:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763282.683785:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763282.683786:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763282.683789:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3260 00000100:00080000:0.0:1789763282.683792:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763282.683795:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763282.683797:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763282.683805:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507771776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763282.683811:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763282.683814:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763282.683816:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763282/real 1789763282] req@ffff8d25d25b9f80 x1876699507771264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763282.683817:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763282.683825:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3260 00000100:00080000:0.0:1789763282.683831:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763282.683836:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763282.683839:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763282.683845:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507771264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763282.683849:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507771904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763282.683869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763282.683875:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763282.683879:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507771264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763282.683904:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507771392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763282.683920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763282.683930:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763282/real 1789763282] req@ffff8d25d25bb800 x1876699507771392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763282.683939:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507771392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763282.683945:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763282.683948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763282.683950:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507771392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763282.683954:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507771520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763282.683965:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763282.683972:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763282/real 1789763282] req@ffff8d25d25ba680 x1876699507771520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763282.683979:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507771520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763282.683985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763282.683986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763282.683988:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507771520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763282.683991:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507771648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763282.684002:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763282.684008:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763282/real 1789763282] req@ffff8d25d25bbb80 x1876699507771648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763282.684015:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507771648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763282.684021:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763282.684038:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763282.684040:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507771648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763282.684045:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763282.684064:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763282.684072:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763282/real 1789763282] req@ffff8d25d25b9180 x1876699507771776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763282.684080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507771776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763282.684086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763282.684089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763282.684091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763282.684095:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763282.684106:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763282.684112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763282/real 1789763282] req@ffff8d25d25b8a80 x1876699507771904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763282.684120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507771904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763307 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763282.684125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763282.684127:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763282.684129:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507771904:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763282.747467:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763283.771568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763284.795485:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763285.819586:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763286.843720:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763287.803504:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763287.803513:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763287.803516:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763287.803524:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3265 00000100:00080000:2.0:1789763287.803531:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763287.803536:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763287.803566:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507772032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763287.803587:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763287.803590:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763287.803592:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763287.803595:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3265 00000100:00080000:2.0:1789763287.803598:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763287.803601:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763287.803609:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763287.803619:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507772032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763287.803621:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507772160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763287.803634:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763287.803636:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763287.803637:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763287.803641:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3265 00000100:00080000:2.0:1789763287.803645:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763287.803649:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763287.803654:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763287.803668:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507772288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763287.803679:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763287.803681:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763287.803686:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763287.803689:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763287.803694:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3265 00000100:00080000:2.0:1789763287.803697:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763287.803700:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763287.803705:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763287.803714:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507772416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763287.803746:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763287/real 1789763287] req@ffff8d25c8023480 x1876699507772032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763287.803751:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763287.803766:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763287.803773:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763287.803783:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3265 00000100:00080000:2.0:1789763287.803790:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763287.803796:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507772032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763287.803802:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763287.803816:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763287.803827:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763287.803840:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763287.803840:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507772544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763287.803856:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507772032:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763287.803871:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763287.803878:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763287.803881:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763287.803887:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3265 00000100:00080000:2.0:1789763287.803891:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763287.803895:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763287.803902:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763287.803906:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763287.803919:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507772672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763287.803952:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763287.803977:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763287/real 1789763287] req@ffff8d25c8021c00 x1876699507772160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763287.804025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507772160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763287.804050:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763287.804052:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763287.804055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763287.804059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763287.804073:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763287.804081:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763287/real 1789763287] req@ffff8d25c8021500 x1876699507772288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763287.804088:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507772288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763287.804093:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763287.804095:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763287.804097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763287.804100:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772416:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763287.804129:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763287.804142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763287/real 1789763287] req@ffff8d25c8020e00 x1876699507772416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763287.804156:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507772416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763287.804166:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763287.804169:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763287.804173:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763287.804179:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507772544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763287.804199:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763287.804210:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763287/real 1789763287] req@ffff8d25c8020700 x1876699507772544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763287.804223:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507772544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763287.804233:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763287.804237:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763287.804241:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507772544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763287.804262:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507772672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763287.804281:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763287.804293:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763287/real 1789763287] req@ffff8d25c8021880 x1876699507772672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763287.804307:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507772672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763312 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763287.804316:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763287.804319:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763287.804323:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507772672:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763287.867508:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763288.891484:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763289.915578:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763290.939569:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763291.963488:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763292.923511:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763292.923521:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763292.923526:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763292.923536:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3270 00000100:00080000:2.0:1789763292.923546:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763292.923553:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763292.923608:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507772800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763292.923633:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763292.923636:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763292.923638:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763292.923641:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3270 00000100:00080000:2.0:1789763292.923644:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763292.923648:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763292.923658:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763292.923668:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507772928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763292.923681:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763292.923688:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763292.923692:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763292.923694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507772800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763292.923697:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3270 00000100:00080000:2.0:1789763292.923704:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763292.923714:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763292.923718:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763292.923725:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507773056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763292.923738:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763292.923741:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763292.923742:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763292.923746:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3270 00000100:00080000:2.0:1789763292.923748:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763292.923751:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763292.923754:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763292.923763:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507773184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763292.923770:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763292.923773:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763292.923774:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763292.923777:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3270 00000100:00080000:2.0:1789763292.923780:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763292.923782:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763292.923786:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763292.923794:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507773312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763292.923803:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763292.923818:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763292.923821:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763292.923824:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763292.923828:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3270 00000100:00080000:2.0:1789763292.923832:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763292.923837:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763292.923844:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763292.923852:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507773440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763292.923886:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763292/real 1789763292] req@ffff8d25c8021880 x1876699507772800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763292.923907:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507772800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763292.923917:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763292.923921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763292.923925:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507772800:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763292.923973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763292.924021:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763292.924036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763292/real 1789763292] req@ffff8d25c8020700 x1876699507772928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763292.924047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507772928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763292.924054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763292.924057:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763292.924060:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507772928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763292.924065:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763292.924081:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763292.924089:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763292/real 1789763292] req@ffff8d25c8020e00 x1876699507773056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763292.924098:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507773056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763292.924104:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763292.924106:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763292.924109:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763292.924113:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763292.924125:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763292.924132:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763292/real 1789763292] req@ffff8d25c8021500 x1876699507773184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763292.924141:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507773184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763292.924147:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763292.924149:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763292.924151:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763292.924155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507773312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763292.924178:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763292.924185:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763292/real 1789763292] req@ffff8d25c8021c00 x1876699507773312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763292.924194:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507773312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763292.924225:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763292.924228:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763292.924231:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507773312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763292.924236:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507773440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763292.924253:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763292.924266:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763292/real 1789763292] req@ffff8d25c8023480 x1876699507773440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763292.924275:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507773440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763317 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763292.924282:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763292.924285:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763292.924288:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507773440:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763292.987561:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763294.011499:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763295.035496:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763296.059492:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763297.083469:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763298.043458:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.043466:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763298.043470:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.043479:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3275 00000100:00080000:2.0:1789763298.043487:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.043494:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763298.043527:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507773568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763298.043555:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.043558:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763298.043560:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.043563:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3275 00000100:00080000:2.0:1789763298.043566:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.043569:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763298.043578:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763298.043587:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507773568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763298.043592:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507773696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763298.043602:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.043605:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763298.043606:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.043609:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3275 00000100:00080000:2.0:1789763298.043612:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.043614:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763298.043617:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763298.043625:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507773824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763298.043631:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.043634:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763298.043635:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.043638:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3275 00000100:00080000:2.0:1789763298.043640:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.043642:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763298.043645:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763298.043651:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763298.043656:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507773952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763298.043665:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.043667:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763298.043669:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763298.043672:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3275 00000100:00080000:2.0:1789763298.043675:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.043677:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763298.043680:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763298.043688:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507774080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763298.043694:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763298.043696:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763298.043700:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763298.043701:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffff8d25c8023100 x1876699507773568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763298.043705:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3275 00000100:00080000:2.0:1789763298.043711:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763298.043718:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763298.043722:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763298.043729:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507773568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763298.043731:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507774208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763298.043756:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763298.043761:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763298.043765:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507773568:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763298.043786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763298.043803:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763298.043813:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffff8d25c8022d80 x1876699507773696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763298.043821:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507773696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763298.043827:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763298.043829:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763298.043831:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763298.043836:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763298.043846:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763298.043852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffff8d25c8020380 x1876699507773824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763298.043860:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507773824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763298.043865:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763298.043866:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763298.043868:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763298.043871:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763298.043881:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763298.043887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffff8d25c8022300 x1876699507773952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763298.043894:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507773952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763298.043899:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763298.043900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763298.043902:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507773952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763298.043905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763298.043935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763298.043943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffff8d25c8023b80 x1876699507774080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763298.043951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507774080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763298.043957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763298.043959:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763298.043961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763298.043964:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763298.043974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763298.043988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763298/real 1789763298] req@ffff8d25c8022a00 x1876699507774208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763298.043998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507774208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763323 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763298.044007:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763298.044010:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763298.044013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774208:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763298.107491:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763299.131492:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763300.155493:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763301.179498:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763302.203515:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763303.163496:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763303.163507:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763303.163511:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763303.163521:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3280 00000100:00080000:3.0:1789763303.163532:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763303.163540:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763303.163584:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507774336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763303.163611:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763303.163614:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763303.163616:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763303.163620:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3280 00000100:00080000:3.0:1789763303.163622:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763303.163626:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763303.163637:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763303.163647:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507774464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763303.163658:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763303.163664:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763303.163667:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763303.163669:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507774336:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763303.163673:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3280 00000100:00080000:3.0:1789763303.163681:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763303.163689:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763303.163693:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763303.163701:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507774592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763303.163708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763303.163711:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763303.163712:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763303.163715:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3280 00000100:00080000:3.0:1789763303.163717:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763303.163720:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763303.163724:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763303.163733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507774720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763303.163740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763303.163743:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763303.163745:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763303.163752:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3280 00000800:00000100:1.0:1789763303.163758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763303.163760:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763303.163768:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763303.163773:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763303.163782:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507774848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763303.163795:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763303.163798:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763303.163799:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763303.163802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3280 00000100:00080000:3.0:1789763303.163805:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763303.163807:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763303.163812:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763303.163823:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507774976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763303.163825:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffff8d25d1b09180 x1876699507774336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763303.163867:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507774336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763303.163901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763303.163907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763303.163912:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507774336:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763303.163941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507774464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763303.163969:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763303.163984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffff8d25d1b0aa00 x1876699507774464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763303.163999:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507774464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763303.164008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763303.164012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763303.164016:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507774464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763303.164023:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507774592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763303.164046:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763303.164060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffff8d25d1b09c00 x1876699507774592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763303.164073:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507774592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763303.164083:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763303.164086:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763303.164090:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507774592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763303.164098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507774720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763303.164120:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763303.164133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffff8d25d1b08700 x1876699507774720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763303.164146:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507774720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763303.164156:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763303.164159:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763303.164163:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507774720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763303.164170:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763303.164192:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763303.164205:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffff8d25d1b0a680 x1876699507774848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763303.164218:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507774848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763303.164226:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763303.164230:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763303.164233:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763303.164240:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763303.164261:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763303.164274:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763303/real 1789763303] req@ffff8d25d1b0bb80 x1876699507774976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763303.164288:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507774976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763328 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763303.164297:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763303.164300:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763303.164304:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507774976:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763303.227489:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763304.251488:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763305.275477:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763306.299478:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763307.323469:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763308.283437:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763308.283444:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763308.283448:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763308.283457:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3285 00000100:00080000:3.0:1789763308.283466:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763308.283473:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763308.283504:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507775104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763308.283527:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763308.283530:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763308.283532:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763308.283535:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3285 00000100:00080000:3.0:1789763308.283538:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763308.283541:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763308.283550:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763308.283560:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507775104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763308.283564:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507775232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763308.283575:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763308.283578:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763308.283580:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763308.283583:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3285 00000100:00080000:3.0:1789763308.283586:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763308.283588:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763308.283592:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763308.283600:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507775360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763308.283607:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763308.283609:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763308.283610:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763308.283613:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3285 00000800:00000100:0.0:1789763308.283618:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763308.283619:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763308.283624:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763308.283628:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763308.283637:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507775488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763308.283643:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763308.283646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763308.283647:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763308.283650:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3285 00000100:00080000:3.0:1789763308.283653:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763308.283657:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffff8d25d1b0b800 x1876699507775104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763308.283660:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763308.283675:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507775104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763308.283684:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763308.283687:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763308.283690:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507775104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763308.283708:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507775232:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789763308.283718:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763308.283727:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763308.283732:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507775616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763308.283740:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffff8d25d1b09880 x1876699507775232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763308.283743:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763308.283751:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763308.283755:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763308.283762:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507775232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763308.283762:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3285 00000100:00080000:3.0:1789763308.283768:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763308.283775:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763308.283783:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763308.283786:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763308.283789:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507775232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763308.283792:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507775360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763308.283805:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763308.283811:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffff8d25d1b08e00 x1876699507775360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763308.283819:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507775360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763308.283824:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763308.283826:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763308.283829:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507775360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763308.283833:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507775488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763308.283857:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763308.283866:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffff8d25d1b0bb80 x1876699507775488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763308.283874:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507775488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763308.283883:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763308.283884:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763308.283890:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763308.283894:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507775488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763308.283901:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507775744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763308.283915:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507775616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763308.283930:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763308.283942:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffff8d25d1b0a680 x1876699507775616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763308.283959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507775616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763308.283966:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763308.283969:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763308.283972:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507775616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763308.283985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507775744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763308.283996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763308.284002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763308/real 1789763308] req@ffff8d25d1b08700 x1876699507775744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763308.284010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507775744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763333 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763308.284015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763308.284016:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763308.284018:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507775744:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763308.347486:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763309.371469:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763310.395481:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763311.419517:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763312.443546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763313.403616:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763313.403628:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763313.403633:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763313.403645:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3291 00000100:00080000:1.0:1789763313.403672:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763313.403685:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763313.403741:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507775872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763313.403817:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763313.403826:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763313.403830:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763313.403854:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3291 00000100:00080000:1.0:1789763313.403859:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763313.403865:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763313.403875:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763313.403901:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507776000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763313.403917:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763313.403920:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763313.403923:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763313.403926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507775872:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763313.403930:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3291 00000100:00080000:1.0:1789763313.403939:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763313.403942:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763313.403946:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763313.403958:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507776128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763313.403986:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763313.403991:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763313.403994:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763313.404000:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3291 00000100:00080000:1.0:1789763313.404017:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763313.404025:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789763313.404034:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:1.0:1789763313.404040:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763313.404063:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507776256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763313.404076:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763313.404081:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763313.404083:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763313.404089:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3291 00000100:00080000:1.0:1789763313.404096:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763313.404105:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763313/real 1789763313] req@ffff8d25c8168700 x1876699507775872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763313.404108:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763313.404120:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763313.404145:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507776384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763313.404152:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507775872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763313.404179:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763313.404188:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763313.404191:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763313.404192:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763313.404210:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3291 00000100:00080000:1.0:1789763313.404214:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763313.404217:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763313.404222:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763313.404225:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763313.404237:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507775872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763313.404240:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507776512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763313.404293:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763313.404322:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763313.404336:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763313/real 1789763313] req@ffff8d25c8168380 x1876699507776000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763313.404349:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507776000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763313.404357:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763313.404360:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763313.404363:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763313.404369:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763313.404383:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763313.404451:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763313/real 1789763313] req@ffff8d25c8168e00 x1876699507776128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763313.404461:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507776128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763313.404472:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763313.404475:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763313.404477:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763313.404481:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763313.404497:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763313.404505:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763313/real 1789763313] req@ffff8d25c8169f80 x1876699507776256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763313.404513:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507776256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763313.404519:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763313.404521:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763313.404523:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776256:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763313.404527:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507776384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763313.404538:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763313.404545:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763313/real 1789763313] req@ffff8d25c8168000 x1876699507776384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763313.404554:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507776384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763313.404559:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763313.404561:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763313.404564:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507776384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763313.404568:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507776512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763313.404579:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763313.404585:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763313/real 1789763313] req@ffff8d25c8169500 x1876699507776512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763313.404593:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507776512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763338 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763313.404599:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763313.404600:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763313.404602:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507776512:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763313.467578:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763314.491637:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763315.515634:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763316.539489:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763317.563508:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763318.523603:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763318.523625:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763318.523633:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763318.523654:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3296 00000100:00080000:3.0:1789763318.523671:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763318.523683:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763318.523783:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507776640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763318.523838:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763318.523844:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763318.523847:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763318.523854:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3296 00000100:00080000:3.0:1789763318.523860:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763318.523868:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763318.523884:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763318.523908:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507776768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763318.523917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507776640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763318.523930:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763318.523937:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763318.523939:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763318.523947:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3296 00000100:00080000:3.0:1789763318.523951:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763318.523957:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763318.523964:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763318.523977:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507776896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763318.523990:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763318.523995:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763318.523998:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763318.524003:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3296 00000100:00080000:3.0:1789763318.524007:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763318.524011:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763318.524019:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763318.524035:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507777024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763318.524040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763318.524073:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763318.524081:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763318.524085:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763318.524095:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3296 00000100:00080000:3.0:1789763318.524103:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763318.524112:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763318.524123:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763318.524147:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763318/real 1789763318] req@ffff8d25d1b09c00 x1876699507776640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763318.524154:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507777152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763318.524202:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763318.524208:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763318.524211:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763318.524219:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507776640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763318.524224:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3296 00000100:00080000:3.0:1789763318.524230:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763318.524238:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763318.524252:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763318.524262:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763318.524275:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507777280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763318.524276:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763318.524291:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507776640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763318.524350:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763318.524435:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763318.524458:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763318/real 1789763318] req@ffff8d25d1b0aa00 x1876699507776768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763318.524475:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507776768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763318.524487:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763318.524504:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763318.524510:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763318.524518:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763318.524545:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763318.524559:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763318/real 1789763318] req@ffff8d25d1b09180 x1876699507776896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763318.524574:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507776896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763318.524585:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763318.524589:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763318.524593:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507776896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763318.524600:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763318.524620:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763318.524631:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763318/real 1789763318] req@ffff8d25d1b0ad80 x1876699507777024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763318.524646:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507777024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763318.524657:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763318.524661:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763318.524665:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763318.524671:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507777152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763318.524691:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763318.524702:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763318/real 1789763318] req@ffff8d25d1b08a80 x1876699507777152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763318.524717:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507777152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763318.524734:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763318.524740:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763318.524746:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507777152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763318.524783:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507777280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763318.524826:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763318.524849:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763318/real 1789763318] req@ffff8d25d1b0b480 x1876699507777280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763318.524866:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507777280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763343 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763318.524876:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763318.524880:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763318.524885:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507777280:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763318.587547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763319.611592:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763320.635547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763321.659570:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763322.683585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763323.643597:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763323.643612:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763323.643618:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763323.643631:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3301 00000100:00080000:0.0:1789763323.643644:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763323.643653:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763323.643700:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507777408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763323.643768:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763323.643773:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763323.643776:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763323.643782:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3301 00000100:00080000:0.0:1789763323.643787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763323.643792:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763323.643804:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763323.643821:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507777536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763323.643836:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763323.643841:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763323.643844:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763323.643849:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3301 00000100:00080000:0.0:1789763323.643854:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763323.643861:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763323.643868:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763323.643880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507777408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763323.643887:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507777664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763323.643910:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763323.643914:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763323.643916:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763323.643922:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3301 00000100:00080000:0.0:1789763323.643926:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763323.643931:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763323.643938:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763323.643955:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507777792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763323.643968:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763323.643972:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763323.643975:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763323.643978:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3301 00000100:00080000:0.0:1789763323.643984:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763323.643992:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763323.643996:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763323.644005:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763323.644022:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507777920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763323.644034:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763323.644040:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763323.644043:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763323.644050:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3301 00000100:00080000:0.0:1789763323.644056:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763323.644064:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763323/real 1789763323] req@ffff8d25d25bad80 x1876699507777408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763323.644067:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763323.644081:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763323.644106:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507778048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763323.644113:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507777408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763323.644130:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763323.644141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763323.644150:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507777408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763323.644163:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763323.644194:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763323.644210:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763323/real 1789763323] req@ffff8d25d25ba300 x1876699507777536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763323.644225:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507777536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763323.644236:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763323.644240:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763323.644244:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763323.644278:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763323.644299:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763323.644311:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763323/real 1789763323] req@ffff8d25d25b8000 x1876699507777664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763323.644326:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507777664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763323.644336:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763323.644339:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763323.644343:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763323.644351:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763323.644369:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763323.644435:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763323/real 1789763323] req@ffff8d25d25b8a80 x1876699507777792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763323.644451:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507777792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763323.644462:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763323.644466:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763323.644470:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507777792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763323.644477:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507777920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763323.644501:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763323.644514:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763323/real 1789763323] req@ffff8d25d25b9180 x1876699507777920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763323.644528:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507777920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763323.644538:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763323.644541:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763323.644545:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507777920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763323.644551:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507778048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763323.644570:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763323.644581:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763323/real 1789763323] req@ffff8d25d25bbb80 x1876699507778048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763323.644595:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507778048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763348 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763323.644613:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763323.644617:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763323.644621:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507778048:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763323.707625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763324.731563:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763325.755540:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763326.779615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763327.803527:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763328.763460:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763328.763471:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763328.763475:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763328.763484:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3306 00000100:00080000:0.0:1789763328.763494:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763328.763501:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763328.763537:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507778176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763328.763565:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763328.763569:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763328.763571:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763328.763576:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3306 00000100:00080000:0.0:1789763328.763579:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763328.763584:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763328.763593:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763328.763608:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507778304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763328.763620:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763328.763624:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763328.763626:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763328.763631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507778176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763328.763633:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3306 00000100:00080000:0.0:1789763328.763640:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763328.763646:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763328.763652:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763328.763665:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507778432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763328.763674:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763328.763678:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763328.763680:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763328.763700:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3306 00000100:00080000:0.0:1789763328.763706:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763328.763710:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763328.763716:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763328.763728:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507778560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763328.763740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763328.763744:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763328.763746:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763328.763752:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3306 00000100:00100000:1.0:1789763328.763757:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507778304:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763328.763759:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763328.763768:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763328.763775:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763328.763790:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507778688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763328.763800:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507778432:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763328.763804:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763328.763809:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763328.763811:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763328.763817:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3306 00000100:00080000:0.0:1789763328.763823:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763328.763827:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763328.763832:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507778560:192.168.203.141@tcp:8:kworker.0 00000008:00100000:0.0:1789763328.763835:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763328.763852:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507778816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763328.763872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507778688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763328.763899:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507778816:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763328.827484:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763329.851540:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763330.875504:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763331.835863:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789763331.835930:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763331.835950:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763331.836026:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763331.836045:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763331.836062:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763331.836083:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763331.836101:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789763331.836112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763328/real 1789763331] req@ffff8d25d25bb800 x1876699507778304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763331.836166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507778304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763331.836189:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763331.836199:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763331.836207:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507778304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763331.836221:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763328/real 1789763331] req@ffff8d25d25b9f80 x1876699507778432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763331.836247:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507778432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763331.836263:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763331.836270:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763331.836277:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507778432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763331.836302:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763328/real 1789763331] req@ffff8d25d25bb480 x1876699507778560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763331.836326:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507778560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763331.836341:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763331.836349:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763331.836356:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507778560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763331.836368:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763328/real 1789763331] req@ffff8d25d25ba680 x1876699507778176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763331.836448:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507778176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763331.836466:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763331.836474:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763331.836481:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507778176:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763331.836545:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763328/real 1789763331] req@ffff8d25d25bb100 x1876699507778688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763331.836577:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507778688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763331.836592:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763331.836599:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763331.836606:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507778688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763331.836617:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763328/real 1789763331] req@ffff8d25d25b9c00 x1876699507778816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763331.836647:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507778816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763353 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763331.836691:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763331.836700:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763331.836707:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507778816:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763331.899575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763332.923519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763333.883713:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763333.883726:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763333.883731:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763333.883744:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3311 00000100:00080000:0.0:1789763333.883755:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763333.883764:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763333.883807:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507778944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763333.883848:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763333.883854:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763333.883857:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763333.883862:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3311 00000100:00080000:0.0:1789763333.883866:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763333.883871:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763333.883883:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763333.883899:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507779072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763333.883918:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763333.883926:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763333.883934:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507778944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763333.883935:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763333.883957:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3311 00000100:00080000:0.0:1789763333.883965:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763333.883973:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763333.883984:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763333.884011:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507779200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763333.884033:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763333.884044:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763333.884047:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763333.884055:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763333.884064:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3311 00000100:00080000:0.0:1789763333.884086:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763333.884095:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763333.884117:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763333.884123:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763333/real 1789763333] req@ffff8d25d25b9c00 x1876699507778944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763333.884150:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507779328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763333.884174:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763333.884181:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763333.884186:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763333.884197:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507778944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763333.884202:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3311 00000100:00080000:0.0:1789763333.884216:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763333.884234:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763333.884240:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763333.884246:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763333.884258:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763333.884270:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507778944:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763333.884280:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507779456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763333.884303:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763333.884310:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763333.884313:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763333.884330:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779072:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763333.884348:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3311 00000100:00080000:0.0:1789763333.884360:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763333.884370:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763333.884425:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763333.884450:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507779584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763333.884488:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763333.884527:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763333/real 1789763333] req@ffff8d25d25bb100 x1876699507779072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763333.884559:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507779072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763333.884577:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763333.884583:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763333.884591:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763333.884604:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763333.884692:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763333.884719:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763333/real 1789763333] req@ffff8d25d25ba680 x1876699507779200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763333.884746:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507779200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763333.884777:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763333.884782:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763333.884787:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763333.884796:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763333.884834:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763333.884857:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763333/real 1789763333] req@ffff8d25d25bb480 x1876699507779328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763333.884881:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507779328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763333.884900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763333.884910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763333.884919:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763333.884934:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507779456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763333.884976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763333.885000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763333/real 1789763333] req@ffff8d25d25b9f80 x1876699507779456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763333.885022:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507779456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763333.885037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763333.885044:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763333.885051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507779456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763333.885090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507779584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763333.885132:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763333.885157:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763333/real 1789763333] req@ffff8d25d25bb800 x1876699507779584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763333.885181:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507779584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763358 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763333.885197:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763333.885203:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763333.885210:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507779584:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763333.947589:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763334.971488:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763335.995572:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763337.019550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763338.043493:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763339.003678:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763339.003696:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763339.003705:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763339.003726:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3316 00000100:00080000:2.0:1789763339.003742:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763339.003754:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763339.003815:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507779712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763339.003859:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763339.003864:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763339.003866:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763339.003872:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3316 00000100:00080000:2.0:1789763339.003875:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763339.003882:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763339.003897:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763339.003913:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507779840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763339.003928:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763339.003933:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763339.003936:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763339.003948:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3316 00000100:00100000:1.0:1789763339.003951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507779712:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763339.003953:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763339.003971:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763339.003979:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763339.004009:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507779968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763339.004022:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763339.004028:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763339.004031:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763339.004035:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3316 00000100:00080000:2.0:1789763339.004040:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763339.004046:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763339.004053:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763339.004061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763339.004069:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507780096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763339.004085:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763339.004091:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763339.004093:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763339.004100:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3316 00000100:00080000:2.0:1789763339.004106:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763339.004111:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763339.004119:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763339.004131:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507780224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763339.004141:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763339.004145:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763339.004147:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763339.004153:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3316 00000100:00080000:2.0:1789763339.004158:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763339.004162:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763339.004169:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763339.004176:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffff8d25c8020000 x1876699507779712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763339.004188:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507780352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763339.004285:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507779712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763339.004315:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763339.004322:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763339.004327:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507779712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763339.004363:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763339.004446:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763339.004467:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffff8d25c8023800 x1876699507779840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763339.004484:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507779840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763339.004496:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763339.004500:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763339.004504:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763339.004512:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763339.004534:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763339.004548:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffff8d25c8022680 x1876699507779968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763339.004563:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507779968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763339.004572:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763339.004576:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763339.004580:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507779968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763339.004587:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763339.004606:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763339.004653:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffff8d25c8021180 x1876699507780096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763339.004668:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507780096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763339.004683:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763339.004690:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763339.004697:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763339.004709:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507780224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763339.004750:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763339.004773:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffff8d25c8020a80 x1876699507780224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763339.004795:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507780224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763339.004806:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763339.004810:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763339.004814:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507780224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763339.004822:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507780352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763339.004850:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763339.004864:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763339/real 1789763339] req@ffff8d25c8022a00 x1876699507780352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763339.004878:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507780352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763364 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763339.004887:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763339.004891:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763339.004896:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507780352:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763339.067590:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763340.091665:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763341.115644:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763342.139559:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763343.163464:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763344.123547:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763344.123560:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763344.123567:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763344.123610:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3321 00000100:00080000:2.0:1789763344.123624:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763344.123634:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763344.123686:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507780480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763344.123726:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763344.123732:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763344.123736:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763344.123744:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3321 00000100:00080000:2.0:1789763344.123752:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763344.123758:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763344.123772:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763344.123795:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507780608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763344.123808:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507780480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763344.123817:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763344.123839:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763344.123846:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763344.123854:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3321 00000100:00080000:2.0:1789763344.123861:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763344.123870:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763344.123879:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763344.123912:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507780736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763344.123925:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763344.123931:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763344.123935:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763344.123945:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3321 00000100:00080000:2.0:1789763344.123952:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763344.123957:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763344.123965:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763344.123983:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507780864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763344.123996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763344.124021:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763344.124027:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763344.124031:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763344.124050:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3321 00000100:00080000:2.0:1789763344.124055:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763344.124060:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763344.124066:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763344.124082:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507780992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763344.124096:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763344.124101:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763344.124108:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763344.124110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffff8d25c8023b80 x1876699507780480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763344.124117:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3321 00000100:00080000:2.0:1789763344.124132:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763344.124149:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763344.124180:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763344.124196:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507780480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763344.124206:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507781120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763344.124306:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763344.124317:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763344.124326:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507780480:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763344.124372:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763344.124519:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763344.124563:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffff8d25c8022300 x1876699507780608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763344.124646:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507780608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763344.124660:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763344.124668:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763344.124673:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763344.124685:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763344.124719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763344.124733:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffff8d25c8020380 x1876699507780736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763344.124754:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507780736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763344.124764:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763344.124767:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763344.124771:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763344.124777:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763344.124796:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763344.124806:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffff8d25c8022d80 x1876699507780864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763344.124819:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507780864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763344.124828:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763344.124831:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763344.124834:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507780864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763344.124840:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507780992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763344.124858:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763344.124868:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffff8d25c8023100 x1876699507780992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763344.124881:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507780992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763344.124926:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763344.124930:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763344.124934:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507780992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763344.124941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507781120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763344.124964:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763344.124976:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763344/real 1789763344] req@ffff8d25c8023480 x1876699507781120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763344.124990:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507781120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763369 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763344.125000:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763344.125003:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763344.125006:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507781120:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763344.187564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763345.211598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763346.235599:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763347.259569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763348.283624:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763349.243496:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763349.243521:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763349.243527:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763349.243566:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3326 00000100:00080000:2.0:1789763349.243580:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763349.243591:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763349.243638:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507781248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763349.243679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763349.243684:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763349.243686:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763349.243692:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3326 00000100:00080000:2.0:1789763349.243696:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763349.243701:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763349.243715:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763349.243733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507781376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763349.243751:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763349.243753:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507781248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763349.243763:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763349.243770:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763349.243776:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3326 00000100:00080000:2.0:1789763349.243780:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763349.243784:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763349.243791:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763349.243809:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507781504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763349.243831:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763349.243835:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763349.243838:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763349.243844:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3326 00000100:00080000:2.0:1789763349.243849:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763349.243853:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763349.243860:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789763349.243861:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763349.243885:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507781632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763349.243900:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763349.243904:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763349.243907:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763349.243913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3326 00000100:00080000:2.0:1789763349.243919:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763349.243928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763349] req@ffff8d25c8021c00 x1876699507781248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763349.243939:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763349.243956:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763349.243983:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507781760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763349.243997:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763349.244001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507781248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763349.244004:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763349.244013:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763349.244034:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763349.244036:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3326 00000100:00080000:0.0:1789763349.244050:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763349.244052:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763349.244063:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507781248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763349.244073:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763349.244088:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763349.244104:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507781376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763349.244121:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507781888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763349.244146:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763349.244170:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763349] req@ffff8d25c8021500 x1876699507781376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763349.244188:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507781376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763349.244199:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763349.244203:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763349.244207:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507781376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763349.244214:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507781504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763349.244234:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763349.244247:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763349] req@ffff8d25c8020e00 x1876699507781504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763349.244270:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507781504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763349.244287:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763349.244293:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763349.244300:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507781504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763349.244312:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507781632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763349.244365:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763349.244431:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763349] req@ffff8d25c8020700 x1876699507781632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763349.244448:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507781632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763349.244459:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763349.244462:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763349.244467:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507781632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763349.244474:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507781760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763349.244503:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763349.244516:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763349] req@ffff8d25c8021880 x1876699507781760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763349.244574:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507781760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763349.244584:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763349.244588:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763349.244592:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507781760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763349.244628:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507781888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763349.244655:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763349.244669:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763349/real 1789763349] req@ffff8d25c8021c00 x1876699507781888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763349.244683:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507781888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763374 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763349.244694:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763349.244698:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763349.244702:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507781888:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763349.307559:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763350.331615:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763351.355607:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763352.379609:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763353.403634:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763354.363625:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763354.363637:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763354.363642:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763354.363655:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3332 00000100:00080000:0.0:1789763354.363667:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763354.363675:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763354.363721:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507782016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763354.363761:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763354.363766:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763354.363769:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763354.363774:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3332 00000100:00080000:0.0:1789763354.363778:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763354.363784:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763354.363796:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763354.363814:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507782144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763354.363825:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507782016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763354.363836:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763354.363841:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763354.363844:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763354.363851:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3332 00000100:00080000:0.0:1789763354.363857:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763354.363862:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763354.363868:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763354.363886:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507782272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763354.363898:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763354.363903:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763354.363905:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763354.363909:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3332 00000100:00080000:0.0:1789763354.363914:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763354.363920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763354.363926:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763354.363935:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763354.363953:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507782400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763354.363967:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763354.363971:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763354.363974:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763354.363980:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3332 00000100:00080000:0.0:1789763354.363986:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763354.363991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffff8d25d25b8e00 x1876699507782016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763354.363995:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763354.364004:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763354.364027:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507782528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763354.364042:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763354.364046:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763354.364054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507782016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763354.364065:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763354.364080:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3332 00000100:00080000:0.0:1789763354.364090:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763354.364098:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763354.364099:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763354.364108:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763354.364113:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763354.364123:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507782016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763354.364132:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507782656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763354.364190:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763354.364244:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763354.364269:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffff8d25d25b8380 x1876699507782144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763354.364287:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507782144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763354.364298:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763354.364303:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763354.364307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763354.364315:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763354.364347:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763354.364361:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffff8d25d25b8700 x1876699507782272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763354.364408:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507782272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763354.364420:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763354.364424:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763354.364428:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763354.364436:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782400:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763354.364461:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763354.364476:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffff8d25d25bb800 x1876699507782400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763354.364491:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507782400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763354.364534:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763354.364549:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763354.364553:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763354.364561:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507782528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763354.364589:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763354.364604:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffff8d25d25b9f80 x1876699507782528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763354.364619:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507782528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763354.364645:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763354.364649:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763354.364653:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507782528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763354.364660:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507782656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763354.364682:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763354.364694:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763354/real 1789763354] req@ffff8d25d25bb480 x1876699507782656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763354.364708:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507782656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763379 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763354.364718:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763354.364721:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763354.364725:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507782656:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763354.427538:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763355.451626:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763356.475597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763357.499558:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763358.523635:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763359.483557:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763359.483570:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763359.483576:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763359.483605:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3337 00000100:00080000:0.0:1789763359.483618:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763359.483628:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763359.483677:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507782784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763359.483714:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763359.483719:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763359.483722:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763359.483728:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3337 00000100:00080000:0.0:1789763359.483733:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763359.483741:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763359.483754:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763359.483774:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507782784:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763359.483776:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507782912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763359.483802:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763359.483806:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763359.483809:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763359.483814:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3337 00000100:00080000:0.0:1789763359.483818:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763359.483823:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763359.483830:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763359.483845:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507783040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763359.483858:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763359.483863:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763359.483866:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763359.483876:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763359.483882:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3337 00000100:00080000:0.0:1789763359.483891:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763359.483898:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763359.483906:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763359.483919:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507783168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763359.483931:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763359.483936:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763359.483939:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763359.483947:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3337 00000100:00080000:0.0:1789763359.483956:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763359.483962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763359/real 1789763359] req@ffff8d25d25ba680 x1876699507782784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763359.483972:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763359.483994:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763359.484006:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507782784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763359.484024:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763359.484027:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507783296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763359.484035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763359.484050:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507782784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763359.484075:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763359.484080:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763359.484083:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763359.484090:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3337 00000100:00080000:0.0:1789763359.484097:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763359.484098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782912:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763359.484110:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763359.484121:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763359.484141:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763359.484149:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507783424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763359.484170:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763359/real 1789763359] req@ffff8d25d25bb100 x1876699507782912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763359.484194:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507782912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763359.484206:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763359.484211:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763359.484215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507782912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763359.484222:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763359.484243:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763359.484256:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763359/real 1789763359] req@ffff8d25d25b9c00 x1876699507783040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763359.484270:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507783040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763359.484280:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763359.484283:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763359.484287:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763359.484294:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763359.484313:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763359.484324:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763359/real 1789763359] req@ffff8d25d25bbb80 x1876699507783168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763359.484338:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507783168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763359.484348:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763359.484352:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763359.484355:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763359.484361:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507783296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763359.484414:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763359.484430:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763359/real 1789763359] req@ffff8d25d25b9180 x1876699507783296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763359.484445:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507783296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763359.484455:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763359.484488:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763359.484493:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507783296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763359.484517:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507783424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763359.484542:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763359.484555:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763359/real 1789763359] req@ffff8d25d25b8a80 x1876699507783424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763359.484570:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507783424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763384 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763359.484581:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763359.484585:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763359.484589:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507783424:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763359.547622:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763360.571520:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763361.595611:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763362.619657:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763363.643616:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763364.603536:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763364.603547:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763364.603552:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763364.603563:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3342 00000100:00080000:0.0:1789763364.603573:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763364.603581:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763364.603623:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507783552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763364.603649:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763364.603653:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763364.603655:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763364.603659:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3342 00000100:00080000:0.0:1789763364.603662:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763364.603666:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763364.603707:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763364.603723:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507783552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763364.603729:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507783680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763364.603750:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763364.603755:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763364.603757:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763364.603762:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3342 00000100:00080000:0.0:1789763364.603766:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763364.603772:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763364.603778:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763364.603793:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507783808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763364.603807:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763364.603811:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763364.603817:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763364.603819:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763364.603829:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3342 00000100:00080000:0.0:1789763364.603838:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763364.603844:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763364.603852:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763364.603870:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507783936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763364.603883:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763364.603889:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763364/real 1789763364] req@ffff8d25d25b8000 x1876699507783552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763364.603891:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763364.603897:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763364.603912:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3342 00000100:00080000:0.0:1789763364.603923:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763364.603931:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763364.603938:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763364.603949:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507783552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763364.603956:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507784064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763364.604024:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763364.604027:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763364.604035:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763364.604045:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763364.604046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763364.604065:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3342 00000100:00100000:1.0:1789763364.604067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507783552:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763364.604083:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763364.604097:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763364.604109:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763364.604115:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763364.604143:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507784192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763364.604167:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763364.604191:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763364/real 1789763364] req@ffff8d25d25ba300 x1876699507783680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763364.604210:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507783680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763364.604222:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763364.604226:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763364.604230:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763364.604239:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763364.604260:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763364.604288:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763364/real 1789763364] req@ffff8d25d25bad80 x1876699507783808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763364.604299:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507783808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763364.604306:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763364.604309:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763364.604312:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763364.604317:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763364.604331:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763364.604340:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763364/real 1789763364] req@ffff8d25d25b8a80 x1876699507783936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763364.604350:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507783936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763364.604358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763364.604360:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763364.604363:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507783936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763364.604368:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763364.604382:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763364.604391:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763364/real 1789763364] req@ffff8d25d25b9180 x1876699507784064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763364.604448:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507784064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763364.604456:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763364.604459:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763364.604462:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763364.604482:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763364.604504:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763364.604515:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763364/real 1789763364] req@ffff8d25d25bbb80 x1876699507784192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763364.604526:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507784192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763389 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763364.604533:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763364.604537:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763364.604539:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784192:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763364.667672:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763365.691497:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763366.715504:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763367.739598:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763368.763601:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763369.723457:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763369.723469:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763369.723474:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763369.723485:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3347 00000100:00080000:0.0:1789763369.723497:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763369.723505:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763369.723550:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507784320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763369.723577:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763369.723581:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763369.723582:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763369.723586:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3347 00000100:00080000:0.0:1789763369.723589:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763369.723593:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763369.723604:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763369.723615:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507784320:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763369.723616:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507784448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763369.723632:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763369.723634:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763369.723635:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763369.723638:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3347 00000100:00080000:0.0:1789763369.723641:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763369.723644:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763369.723648:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763369.723656:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507784576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763369.723663:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763369.723666:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763369.723667:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763369.723671:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3347 00000100:00080000:0.0:1789763369.723690:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763369.723695:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763369.723702:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763369.723702:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763369.723721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507784704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763369.723741:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763369.723744:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763369.723746:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763369.723752:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3347 00000100:00100000:1.0:1789763369.723753:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763369/real 1789763369] req@ffff8d25d25b9c00 x1876699507784320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763369.723758:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763369.723767:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763369.723773:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763369.723780:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507784320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763369.723785:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507784832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763369.723827:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763369.723829:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763369.723830:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763369.723834:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763369.723841:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3347 00000100:00080000:1.0:1789763369.723841:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763369.723843:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763369.723849:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763369.723849:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507784320:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789763369.723857:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763369.723867:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507784960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763369.723886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507784448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763369.723910:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763369.723921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763369/real 1789763369] req@ffff8d25d25bb100 x1876699507784448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763369.723930:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507784448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763369.723937:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763369.723939:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763369.723941:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507784448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763369.723945:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507784576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763369.723956:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763369.723962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763369/real 1789763369] req@ffff8d25d25ba680 x1876699507784576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763369.723970:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507784576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763369.723976:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763369.723978:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763369.723979:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507784576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763369.723984:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507784704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763369.723996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763369.724002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763369/real 1789763369] req@ffff8d25d25bb480 x1876699507784704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763369.724010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507784704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763369.724016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763369.724017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763369.724019:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507784704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763369.724022:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763369.724033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763369.724039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763369/real 1789763369] req@ffff8d25d25b9f80 x1876699507784832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763369.724046:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507784832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763369.724052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763369.724053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763369.724055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763369.724058:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763369.724069:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763369.724074:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763369/real 1789763369] req@ffff8d25d25bb800 x1876699507784960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763369.724083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507784960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763394 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763369.724088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763369.724089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763369.724091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507784960:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763369.787564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763370.811562:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763371.835516:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763372.859632:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763373.883623:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763374.843584:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763374.843596:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763374.843603:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763374.843620:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3352 00000100:00080000:2.0:1789763374.843638:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763374.843650:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763374.843702:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507785088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763374.843744:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763374.843749:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763374.843752:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763374.843757:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3352 00000100:00080000:2.0:1789763374.843761:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763374.843767:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763374.843779:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763374.843795:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507785216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763374.843811:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763374.843816:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763374.843820:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763374.843832:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3352 00000100:00080000:2.0:1789763374.843838:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763374.843842:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507785088:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763374.843846:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763374.843852:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763374.843863:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507785344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763374.843900:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763374.843904:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763374.843907:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763374.843913:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3352 00000100:00080000:2.0:1789763374.843916:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763374.843924:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763374.843929:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763374.843941:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507785472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763374.843955:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763374.843959:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763374.843962:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763374.843967:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3352 00000100:00080000:2.0:1789763374.843972:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763374.843976:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763374.843982:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763374.843993:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507785600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763374.844004:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:1.0:1789763374.844018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763374.844019:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763374.844038:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763374.844052:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3352 00000100:00080000:2.0:1789763374.844059:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763374.844066:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763374.844072:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763374.844093:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507785728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763374.844115:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763374/real 1789763374] req@ffff8d25c8023480 x1876699507785088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763374.844161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507785088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763374.844201:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763374.844208:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763374.844215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507785088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763374.844260:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763374.844300:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763374.844320:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763374/real 1789763374] req@ffff8d25c8023100 x1876699507785216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763374.844337:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507785216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763374.844416:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763374.844422:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763374.844427:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763374.844437:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763374.844471:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763374.844488:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763374/real 1789763374] req@ffff8d25c8022d80 x1876699507785344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763374.844503:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507785344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763374.844514:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763374.844518:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763374.844522:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763374.844530:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763374.844551:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763374.844563:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763374/real 1789763374] req@ffff8d25c8020380 x1876699507785472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763374.844577:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507785472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763374.844588:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763374.844591:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763374.844596:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763374.844604:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507785600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763374.844624:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763374.844636:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763374/real 1789763374] req@ffff8d25c8022300 x1876699507785600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763374.844650:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507785600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763374.844660:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763374.844664:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763374.844668:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507785600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763374.844674:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507785728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763374.844694:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763374.844706:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763374/real 1789763374] req@ffff8d25c8023b80 x1876699507785728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763374.844724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507785728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763399 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763374.844741:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763374.844747:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763374.844754:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507785728:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763374.907591:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763375.931640:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763376.955590:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763377.979596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763379.003549:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763379.963546:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763379.963562:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763379.963568:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763379.963585:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3357 00000100:00080000:0.0:1789763379.963600:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763379.963612:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763379.963665:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507785856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763379.963708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763379.963712:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763379.963715:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763379.963720:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3357 00000100:00080000:0.0:1789763379.963726:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763379.963731:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763379.963746:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763379.963767:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507785984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763379.963784:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763379.963792:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763379.963796:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507785856:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763379.963797:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763379.963821:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3357 00000100:00080000:0.0:1789763379.963826:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763379.963832:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763379.963837:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763379.963850:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507786112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763379.963861:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763379.963865:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763379.963867:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763379.963871:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3357 00000100:00080000:0.0:1789763379.963877:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763379.963882:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763379.963886:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763379.963898:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507786240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763379.963910:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763379.963913:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763379.963915:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763379.963925:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3357 00000800:00000100:2.0:1789763379.963928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763379.963932:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763379.963939:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763379.963944:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763379.963962:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507786368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763379.963972:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763379.963976:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763379.963978:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763379.963986:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3357 00000100:00080000:0.0:1789763379.963993:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763379.963998:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763379.964003:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763379.964007:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763379/real 1789763379] req@ffff8d25d25b8700 x1876699507785856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763379.964018:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507786496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763379.964093:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507785856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763379.964122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763379.964132:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763379.964140:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507785856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763379.964185:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763379.964235:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763379.964258:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763379/real 1789763379] req@ffff8d25d25b8380 x1876699507785984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763379.964281:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507785984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763379.964304:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763379.964339:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763379.964352:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507785984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763379.964370:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763379.964476:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763379.964496:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763379/real 1789763379] req@ffff8d25d25b8e00 x1876699507786112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763379.964512:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507786112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763379.964522:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763379.964527:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763379.964531:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763379.964539:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763379.964561:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763379.964573:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763379/real 1789763379] req@ffff8d25d25bb800 x1876699507786240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763379.964588:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507786240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763379.964598:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763379.964601:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763379.964606:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763379.964614:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507786368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763379.964633:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763379.964645:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763379/real 1789763379] req@ffff8d25d25b9f80 x1876699507786368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763379.964658:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507786368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763379.964668:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763379.964672:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763379.964675:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507786368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763379.964682:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507786496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763379.964701:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763379.964712:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763379/real 1789763379] req@ffff8d25d25bb480 x1876699507786496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763379.964726:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507786496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763404 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763379.964735:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763379.964738:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763379.964742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507786496:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763380.027612:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763381.051778:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763382.075591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763383.099627:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763384.123656:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763385.083540:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763385.083558:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763385.083568:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763385.083587:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3362 00000100:00080000:0.0:1789763385.083602:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763385.083614:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763385.083664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507786624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763385.083702:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763385.083707:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763385.083710:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763385.083715:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3362 00000100:00080000:0.0:1789763385.083720:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763385.083725:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763385.083737:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763385.083752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507786752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763385.083778:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763385.083783:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763385.083786:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763385.083792:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3362 00000100:00080000:0.0:1789763385.083797:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763385.083801:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763385.083807:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763385.083827:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507786880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763385.083829:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507786624:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763385.083864:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763385.083869:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763385.083871:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763385.083876:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3362 00000100:00080000:0.0:1789763385.083880:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763385.083884:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763385.083891:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763385.083906:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507787008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763385.083918:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763385.083922:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763385.083924:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763385.083928:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3362 00000100:00080000:0.0:1789763385.083935:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763385.083939:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763385.083945:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763385.083958:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507787136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763385.083970:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763385.083980:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763385.083985:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763385.083988:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763385.083995:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3362 00000100:00080000:0.0:1789763385.084002:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763385.084008:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763385.084013:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763385.084028:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507787264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763385.084051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffff8d25d25ba680 x1876699507786624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.084080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507786624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.084095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.084101:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763385.084107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507786624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763385.084117:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763385.084149:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763385.084165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffff8d25d25bb100 x1876699507786752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.084180:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507786752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.084190:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.084194:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763385.084198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763385.084229:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763385.084250:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763385.084263:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffff8d25d25b9c00 x1876699507786880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.084320:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507786880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.084331:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.084335:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763385.084339:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507786880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763385.084347:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763385.084426:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763385.084443:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffff8d25d25bbb80 x1876699507787008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.084458:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507787008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.084469:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.084473:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763385.084476:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763385.084483:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507787136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763385.084505:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763385.084518:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffff8d25d25b9180 x1876699507787136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.084531:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507787136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.084550:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.084554:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763385.084558:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507787136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763385.084564:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507787264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763385.084583:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763385.084594:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763385/real 1789763385] req@ffff8d25d25b8a80 x1876699507787264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763385.084608:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507787264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763410 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763385.084618:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763385.084621:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763385.084625:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507787264:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763385.147580:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763386.171632:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763387.195572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763388.219586:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763389.243596:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763390.203548:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.203561:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763390.203566:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.203579:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3367 00000100:00080000:3.0:1789763390.203590:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.203599:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763390.203643:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507787392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763390.203689:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.203695:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763390.203698:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.203703:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3367 00000100:00080000:3.0:1789763390.203707:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.203712:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763390.203723:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763390.203738:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507787520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763390.203750:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.203754:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763390.203758:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.203763:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3367 00000100:00080000:3.0:1789763390.203767:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.203771:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763390.203776:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763390.203785:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507787648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763390.203796:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.203799:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763390.203802:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.203807:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3367 00000100:00080000:3.0:1789763390.203811:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.203815:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763390.203821:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763390.203833:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507787776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763390.203847:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.203854:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763390.203856:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507787392:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763390.203858:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.203874:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3367 00000100:00080000:3.0:1789763390.203879:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.203883:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763390.203889:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763390.203904:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507787904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763390.203915:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763390.203920:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763390.203922:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763390.203927:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3367 00000100:00080000:3.0:1789763390.203933:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763390.203938:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763390.203943:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763390.203953:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763390.203965:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507788032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763390.204012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffff8d25d1b0a300 x1876699507787392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.204039:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507787392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.204053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.204060:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763390.204065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507787392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763390.204075:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763390.204100:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763390.204116:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffff8d25d1b0b100 x1876699507787520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.204131:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507787520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.204141:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.204145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763390.204149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763390.204155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763390.204175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763390.204187:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffff8d25d1b08380 x1876699507787648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.204201:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507787648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.204211:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.204215:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763390.204218:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763390.204284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763390.204314:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763390.204329:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffff8d25d1b09500 x1876699507787776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.204344:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507787776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.204355:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.204359:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763390.204363:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507787776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763390.204369:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507787904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763390.204434:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763390.204449:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffff8d25d1b08000 x1876699507787904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.204464:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507787904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.204474:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.204478:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763390.204482:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507787904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763390.204489:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507788032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763390.204509:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763390.204520:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763390/real 1789763390] req@ffff8d25d1b0b480 x1876699507788032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763390.204543:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507788032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763415 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763390.204552:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763390.204555:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763390.204559:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507788032:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763390.267577:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763391.291585:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763392.315632:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763393.339648:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763394.363609:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763395.323552:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763395.323565:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763395.323571:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763395.323585:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3373 00000100:00080000:2.0:1789763395.323598:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763395.323607:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763395.323654:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507788160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763395.323695:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763395.323699:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763395.323702:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763395.323708:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3373 00000100:00080000:2.0:1789763395.323712:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763395.323719:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763395.323750:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763395.323771:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507788160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763395.323779:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507788288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763395.323803:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763395.323810:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763395.323814:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763395.323822:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3373 00000100:00080000:2.0:1789763395.323830:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763395.323837:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763395.323846:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763395.323864:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507788416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763395.323876:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763395.323880:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763395.323882:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763395.323888:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3373 00000100:00080000:2.0:1789763395.323892:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763395.323898:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763395.323905:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763395.323909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507788288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763395.323928:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507788544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763395.323952:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763395.323958:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763395.323963:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763395.323972:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3373 00000100:00100000:1.0:1789763395.323978:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507788416:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763395.323983:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763395.323997:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763395.324005:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763395.324027:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507788672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763395.324031:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507788544:192.168.203.141@tcp:8:kworker.0 00000100:00080000:2.0:1789763395.324077:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763395.324082:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763395.324086:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763395.324094:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3373 00000100:00080000:2.0:1789763395.324102:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763395.324109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507788672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763395.324115:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763395.324130:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763395.324153:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507788800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763395.324272:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507788800:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763395.387538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763396.411665:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763397.435564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763398.395760:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789763398.395814:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763398.395834:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763398.395887:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763398.395904:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763398.395925:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763398.395947:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763398.395970:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1789763398.395986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763398] req@ffff8d25c8022680 x1876699507788544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763398.396034:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507788544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763398.396056:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763398.396065:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763398.396075:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507788544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763398.396088:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763398] req@ffff8d25c8020a80 x1876699507788288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763398.396122:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507788288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763398.396138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763398.396145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763398.396153:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507788288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763398.396166:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763398] req@ffff8d25c8021180 x1876699507788416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763398.396235:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507788416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763398.396251:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763398.396258:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763398.396265:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507788416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763398.396278:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763398] req@ffff8d25c8022a00 x1876699507788160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763398.396300:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507788160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763398.396314:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763398.396321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763398.396328:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507788160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763398.396373:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763398] req@ffff8d25c8020000 x1876699507788800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763398.396436:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507788800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763398.396453:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763398.396460:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763398.396468:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507788800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763398.396482:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763395/real 1789763398] req@ffff8d25c8023800 x1876699507788672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763398.396508:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507788672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763420 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763398.396523:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763398.396530:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763398.396536:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507788672:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763398.459616:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763399.483662:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763400.443472:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763400.443486:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763400.443492:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763400.443506:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3378 00000100:00080000:2.0:1789763400.443519:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763400.443528:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763400.443577:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507788928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763400.443613:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763400.443617:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763400.443619:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763400.443623:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3378 00000100:00080000:2.0:1789763400.443626:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763400.443631:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763400.443643:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763400.443655:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507789056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763400.443667:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763400.443671:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507788928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763400.443672:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763400.443680:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763400.443698:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3378 00000100:00080000:2.0:1789763400.443701:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763400.443704:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763400.443709:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763400.443721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507789184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763400.443729:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763400.443733:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763400.443734:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763400.443737:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3378 00000100:00080000:2.0:1789763400.443741:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763400.443744:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763400.443748:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763400.443766:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507789312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763400.443780:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763400.443790:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763400.443797:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763400.443801:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763400.443808:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3378 00000100:00080000:2.0:1789763400.443814:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763400.443821:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763400.443828:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763400.443861:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507789440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763400.443862:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763400/real 1789763400] req@ffff8d25c8023800 x1876699507788928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763400.443903:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763400.443927:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763400.443931:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763400.443944:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3378 00000100:00080000:2.0:1789763400.443950:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763400.443956:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763400.443969:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507788928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763400.443971:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:1.0:1789763400.444005:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763400.444006:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507789568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763400.444030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763400.444056:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507788928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763400.444104:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763400.444150:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763400.444198:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763400/real 1789763400] req@ffff8d25c8020000 x1876699507789056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763400.444220:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507789056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763400.444243:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763400.444249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763400.444254:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763400.444264:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763400.444297:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763400.444315:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763400/real 1789763400] req@ffff8d25c8022a00 x1876699507789184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763400.444331:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507789184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763400.444342:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763400.444347:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763400.444352:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763400.444379:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763400.444468:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763400.444486:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763400/real 1789763400] req@ffff8d25c8021180 x1876699507789312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763400.444502:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507789312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763400.444513:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763400.444517:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763400.444522:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763400.444532:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507789440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763400.444561:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763400.444577:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763400/real 1789763400] req@ffff8d25c8020a80 x1876699507789440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763400.444592:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507789440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763400.444603:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763400.444607:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763400.444611:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507789440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763400.444618:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507789568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763400.444645:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763400.444661:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763400/real 1789763400] req@ffff8d25c8022680 x1876699507789568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763400.444679:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507789568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763425 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763400.444689:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763400.444693:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763400.444698:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507789568:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763400.507610:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763401.531592:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763402.555616:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763403.579573:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763404.603538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763405.563597:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763405.563613:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763405.563620:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763405.563638:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3383 00000100:00080000:3.0:1789763405.563665:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763405.563676:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763405.563752:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507789696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763405.563796:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763405.563802:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763405.563805:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763405.563822:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3383 00000100:00080000:3.0:1789763405.563828:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763405.563834:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763405.563849:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763405.563868:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507789824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763405.563880:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507789696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763405.563897:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763405.563912:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763405.563914:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763405.563921:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3383 00000100:00080000:3.0:1789763405.563925:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763405.563930:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763405.563936:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763405.563953:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507789952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763405.563965:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763405.563970:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763405.563973:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763405.563978:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3383 00000100:00080000:3.0:1789763405.563983:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763405.563987:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763405.563994:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763405.564011:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507790080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763405.564024:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763405.564038:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763405.564045:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763405.564048:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763405.564055:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3383 00000100:00080000:3.0:1789763405.564061:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763405.564067:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763405.564073:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763405.564089:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507790208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763405.564102:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763405.564110:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763405/real 1789763405] req@ffff8d25d1b08a80 x1876699507789696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763405.564111:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763405.564142:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763405.564159:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3383 00000100:00080000:3.0:1789763405.564165:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763405.564170:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763405.564183:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763405.564198:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507789696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763405.564209:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507790336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763405.564226:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763405.564237:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763405.564246:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507789696:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763405.564291:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763405.564326:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763405.564345:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763405/real 1789763405] req@ffff8d25d1b0ad80 x1876699507789824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763405.564361:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507789824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763405.564373:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763405.564408:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763405.564414:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763405.564422:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763405.564458:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763405.564473:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763405/real 1789763405] req@ffff8d25d1b09180 x1876699507789952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763405.564488:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507789952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763405.564528:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763405.564535:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763405.564542:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507789952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763405.564555:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763405.564596:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763405.564617:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763405/real 1789763405] req@ffff8d25d1b0aa00 x1876699507790080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763405.564633:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507790080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763405.564645:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763405.564649:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763405.564653:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763405.564661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507790208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763405.564697:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763405.564710:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763405/real 1789763405] req@ffff8d25d1b09c00 x1876699507790208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763405.564724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507790208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763405.564745:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763405.564749:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763405.564754:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507790208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763405.564761:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507790336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763405.564781:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763405.564792:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763405/real 1789763405] req@ffff8d25d1b08700 x1876699507790336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763405.564809:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507790336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763430 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763405.564837:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763405.564843:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763405.564851:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507790336:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763405.627629:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763406.651598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763407.675609:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763408.699479:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763409.723587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763410.683545:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763410.683567:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763410.683575:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763410.683595:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3388 00000100:00080000:2.0:1789763410.683626:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763410.683639:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763410.683703:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507790464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763410.683766:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763410.683774:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763410.683778:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763410.683789:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3388 00000100:00080000:2.0:1789763410.683796:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763410.683805:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763410.683825:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763410.683848:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507790592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763410.683864:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763410.683870:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763410.683874:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763410.683881:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3388 00000100:00080000:2.0:1789763410.683887:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763410.683892:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763410.683900:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763410.683905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507790464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763410.683919:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507790720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763410.683935:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763410.683940:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763410.683942:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763410.683947:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3388 00000100:00080000:2.0:1789763410.683951:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763410.683955:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763410.683961:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763410.683977:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507790848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763410.683990:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763410.683993:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763410.683996:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763410.684001:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3388 00000100:00080000:2.0:1789763410.684007:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763410.684012:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763410.684019:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763410.684025:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763410.684043:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507790976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763410.684060:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763410.684065:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763410.684067:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763410.684072:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3388 00000100:00080000:2.0:1789763410.684076:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763410.684114:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763410.684121:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763410.684142:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507791104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763410.684158:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763410/real 1789763410] req@ffff8d25c8023b80 x1876699507790464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763410.684204:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507790464/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763410.684229:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763410.684238:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763410.684248:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507790464:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763410.684261:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763410.684299:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763410.684321:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763410/real 1789763410] req@ffff8d25c8022300 x1876699507790592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763410.684337:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507790592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763410.684348:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763410.684353:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763410.684357:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763410.684430:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763410.684464:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763410.684489:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763410/real 1789763410] req@ffff8d25c8020380 x1876699507790720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763410.684515:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507790720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763410.684532:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763410.684540:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763410.684548:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763410.684560:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790848:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763410.684595:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763410.684612:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763410/real 1789763410] req@ffff8d25c8022d80 x1876699507790848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763410.684627:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507790848/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763410.684639:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763410.684643:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763410.684647:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507790848:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763410.684654:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507790976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763410.684674:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763410.684687:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763410/real 1789763410] req@ffff8d25c8023100 x1876699507790976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763410.684701:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507790976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763410.684711:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763410.684714:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763410.684717:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507790976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763410.684724:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507791104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763410.684744:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763410.684756:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763410/real 1789763410] req@ffff8d25c8023480 x1876699507791104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763410.684770:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507791104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763435 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763410.684780:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763410.684783:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763410.684788:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507791104:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763410.747671:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763411.771638:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763412.795670:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763413.819595:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763414.843582:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763415.803539:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763415.803556:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763415.803562:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763415.803576:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3393 00000100:00080000:2.0:1789763415.803591:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763415.803601:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763415.803659:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507791232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763415.803700:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763415.803705:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763415.803708:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763415.803714:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3393 00000100:00080000:2.0:1789763415.803722:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763415.803741:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763415.803754:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763415.803783:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507791360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763415.803786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507791232:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763415.803817:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763415.803823:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763415.803826:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763415.803834:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3393 00000100:00080000:2.0:1789763415.803838:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763415.803844:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763415.803851:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763415.803868:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507791488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763415.803881:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763415.803886:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789763415.803893:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763415.803894:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763415.803907:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3393 00000100:00080000:2.0:1789763415.803913:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763415.803918:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763415.803924:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763415.803936:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507791616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763415.803957:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763415.803961:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763415.803963:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763415.803967:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3393 00000100:00080000:2.0:1789763415.803970:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763415.803974:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763415.803979:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763415.803989:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507791744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763415.804001:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763415.804005:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763415.804006:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763415.804011:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3393 00000100:00080000:2.0:1789763415.804017:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763415.804023:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763415.804030:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763415.804075:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507791872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763415.804277:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763415/real 1789763415] req@ffff8d25c8021c00 x1876699507791232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763415.804310:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507791232/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763415.804448:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763415.804458:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763415.804465:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507791232:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763415.804506:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507791360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763415.804558:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763415.804587:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763415/real 1789763415] req@ffff8d25c8021880 x1876699507791360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763415.804608:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507791360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763415.804620:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763415.804624:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763415.804629:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507791360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763415.804637:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507791488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763415.804678:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763415.804695:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763415/real 1789763415] req@ffff8d25c8020700 x1876699507791488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763415.804720:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507791488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763415.804737:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763415.804744:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763415.804750:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507791488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763415.804763:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507791616:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763415.804805:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763415.804823:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763415/real 1789763415] req@ffff8d25c8020e00 x1876699507791616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763415.804842:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507791616/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763415.804858:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763415.804864:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763415.804871:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507791616:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763415.804882:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507791744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763415.804922:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763415.804945:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763415/real 1789763415] req@ffff8d25c8021500 x1876699507791744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763415.804962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507791744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763415.804981:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763415.804985:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763415.804989:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507791744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763415.804997:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507791872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763415.805023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763415.805036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763415/real 1789763415] req@ffff8d25c8023480 x1876699507791872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763415.805085:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507791872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763440 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763415.805102:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763415.805109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763415.805116:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507791872:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763415.867612:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763416.891675:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763417.915629:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763418.939557:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763419.963677:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763420.923505:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763420.923518:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763420.923524:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763420.923537:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3398 00000100:00080000:3.0:1789763420.923549:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763420.923558:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763420.923604:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507792000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763420.923642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763420.923646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763420.923647:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763420.923652:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3398 00000100:00080000:3.0:1789763420.923654:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763420.923657:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763420.923669:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763420.923681:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507792128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763420.923691:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763420.923694:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763420.923696:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763420.923700:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3398 00000100:00080000:3.0:1789763420.923702:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763420.923706:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763420.923711:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763420.923723:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507792256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763420.923739:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507792000:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763420.923740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763420.923748:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763420.923753:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763420.923768:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3398 00000100:00080000:3.0:1789763420.923772:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763420.923776:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763420.923780:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763420.923791:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507792384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763420.923800:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763420.923804:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763420.923806:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763420.923819:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3398 00000100:00080000:3.0:1789763420.923822:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763420.923824:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763420.923829:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763420.923839:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507792512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763420.923848:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763420.923852:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763420.923853:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763420.923857:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3398 00000100:00080000:3.0:1789763420.923859:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763420.923862:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763420.923865:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763420.923878:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507792640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763420.923894:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763420.923985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763420/real 1789763420] req@ffff8d25d1b0a680 x1876699507792000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763420.924053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507792000/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763420.924070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763420.924076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763420.924081:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507792000:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763420.924093:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763420.924130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763420.924148:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763420/real 1789763420] req@ffff8d25d1b0bb80 x1876699507792128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763420.924165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507792128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763420.924175:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763420.924180:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763420.924184:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763420.924219:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763420.924243:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763420.924259:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763420/real 1789763420] req@ffff8d25d1b08e00 x1876699507792256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763420.924274:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507792256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763420.924285:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763420.924289:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763420.924293:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763420.924300:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792384:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763420.924336:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763420.924349:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763420/real 1789763420] req@ffff8d25d1b09880 x1876699507792384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763420.924363:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507792384/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763420.924374:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763420.924418:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763420.924427:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792384:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763420.924440:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507792512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763420.924481:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763420.924506:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763420/real 1789763420] req@ffff8d25d1b0b800 x1876699507792512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763420.924545:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507792512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763420.924556:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763420.924560:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763420.924563:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507792512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763420.924571:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507792640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763420.924596:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763420.924609:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763420/real 1789763420] req@ffff8d25d1b08700 x1876699507792640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763420.924622:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507792640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763445 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763420.924631:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763420.924634:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763420.924638:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507792640:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763420.987684:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763422.011665:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763423.035694:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763424.059618:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763425.083582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763426.043586:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763426.043604:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763426.043611:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763426.043629:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3403 00000100:00080000:2.0:1789763426.043647:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763426.043660:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763426.043735:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507792768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763426.043782:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763426.043788:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763426.043791:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763426.043797:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3403 00000100:00080000:2.0:1789763426.043801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763426.043806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763426.043822:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763426.043841:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507792896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763426.043857:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763426.043862:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763426.043865:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763426.043875:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507792768:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763426.043877:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3403 00000100:00080000:2.0:1789763426.043889:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763426.043897:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763426.043903:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763426.043921:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507793024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763426.043936:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763426.043941:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763426.043945:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763426.043952:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3403 00000100:00080000:2.0:1789763426.043961:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763426.043988:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763426.044000:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763426.044027:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507793152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763426.044035:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763426.044061:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763426.044068:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763426.044072:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763426.044081:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3403 00000100:00080000:2.0:1789763426.044090:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763426.044099:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763426.044109:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763426.044136:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507793280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763426.044141:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffff8d25c8023100 x1876699507792768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763426.044201:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763426.044211:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763426.044216:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763426.044223:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507792768/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763426.044233:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3403 00000100:00080000:2.0:1789763426.044253:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763426.044256:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763426.044273:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763426.044275:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789763426.044292:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763426.044294:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507792768:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763426.044331:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507793408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763426.044358:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763426.044454:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763426.044488:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffff8d25c8022d80 x1876699507792896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763426.044512:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507792896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763426.044525:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763426.044529:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763426.044534:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507792896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763426.044543:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763426.044567:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763426.044589:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffff8d25c8020380 x1876699507793024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763426.044613:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507793024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763426.044624:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763426.044628:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763426.044633:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763426.044645:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793152:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763426.044685:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763426.044709:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffff8d25c8022300 x1876699507793152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763426.044735:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507793152/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763426.044750:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763426.044754:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763426.044759:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793152:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763426.044767:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507793280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763426.044797:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763426.044812:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffff8d25c8023b80 x1876699507793280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763426.044827:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507793280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763426.044837:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763426.044842:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763426.044845:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507793280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763426.044869:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507793408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763426.044889:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763426.044902:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763426/real 1789763426] req@ffff8d25c8022680 x1876699507793408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763426.044916:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507793408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763451 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763426.044927:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763426.044931:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763426.044935:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507793408:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763426.107628:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763427.131628:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763428.155571:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763429.179638:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763430.203567:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763431.163539:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.163552:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763431.163558:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.163571:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3408 00000100:00080000:2.0:1789763431.163586:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.163595:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763431.163641:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507793536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763431.163678:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.163683:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763431.163686:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.163691:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3408 00000100:00080000:2.0:1789763431.163695:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.163699:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763431.163711:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763431.163729:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507793664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763431.163742:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.163747:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763431.163749:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.163754:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3408 00000100:00080000:2.0:1789763431.163759:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.163763:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763431.163769:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763431.163785:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507793536:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763431.163793:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507793792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763431.163816:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.163820:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763431.163822:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.163829:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3408 00000100:00080000:2.0:1789763431.163832:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.163838:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763431.163845:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763431.163861:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507793920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763431.163872:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.163875:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763431.163877:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.163883:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3408 00000100:00080000:2.0:1789763431.163888:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.163892:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763431.163901:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763431.163914:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507794048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763431.163923:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763431.163966:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763431.163973:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763431.163976:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763431.163987:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3408 00000100:00080000:2.0:1789763431.163996:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763431.164002:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763431.164010:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763431.164029:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507794176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763431.164065:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffff8d25c8020a80 x1876699507793536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.164110:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507793536/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.164128:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.164135:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763431.164142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507793536:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763431.164155:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763431.164190:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763431.164207:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffff8d25c8021180 x1876699507793664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.164223:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507793664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.164234:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.164239:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763431.164244:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763431.164295:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763431.164317:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763431.164331:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffff8d25c8022a00 x1876699507793792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.164346:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507793792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.164356:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.164360:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763431.164365:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763431.164372:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793920:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763431.164431:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763431.164445:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffff8d25c8020000 x1876699507793920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.164460:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507793920/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.164471:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.164475:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763431.164479:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507793920:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763431.164486:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763431.164507:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763431.164527:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffff8d25c8023800 x1876699507794048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.164552:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507794048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.164569:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.164575:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763431.164582:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763431.164603:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763431.164640:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763431.164656:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763431/real 1789763431] req@ffff8d25c8022680 x1876699507794176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763431.164675:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507794176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763456 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763431.164691:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763431.164698:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763431.164704:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794176:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763431.227691:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763432.251638:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763433.275645:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763434.299588:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763435.323553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763436.283568:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763436.283583:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763436.283589:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763436.283605:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3413 00000100:00080000:0.0:1789763436.283620:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763436.283631:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763436.283692:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507794304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763436.283729:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763436.283734:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763436.283737:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763436.283742:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3413 00000100:00080000:0.0:1789763436.283747:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763436.283753:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763436.283768:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763436.283784:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507794304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763436.283786:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507794432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763436.283819:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763436.283824:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763436.283826:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763436.283831:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3413 00000100:00080000:0.0:1789763436.283834:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763436.283839:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763436.283846:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763436.283860:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507794560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763436.283871:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763436.283875:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763436.283878:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763436.283882:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3413 00000100:00080000:0.0:1789763436.283887:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763436.283890:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789763436.283920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763436.283922:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763436.283945:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507794688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763436.283959:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763436.283963:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763436.283965:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763436.283969:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3413 00000100:00080000:0.0:1789763436.283975:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763436.283980:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763436.283986:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763436.284002:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507794816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763436.284002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffff8d25d25bad80 x1876699507794304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763436.284040:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763436.284053:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763436.284056:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763436.284060:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3413 00000100:00080000:0.0:1789763436.284082:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763436.284088:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763436.284096:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507794304/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763436.284099:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763436.284123:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507794944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763436.284148:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763436.284158:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763436.284166:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507794304:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763436.284207:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507794432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763436.284269:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763436.284289:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffff8d25d25ba300 x1876699507794432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763436.284306:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507794432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763436.284317:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763436.284322:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763436.284326:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507794432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763436.284334:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507794560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763436.284383:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763436.284442:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffff8d25d25b8000 x1876699507794560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763436.284465:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507794560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763436.284477:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763436.284482:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763436.284487:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507794560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763436.284496:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507794688:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763436.284524:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763436.284537:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffff8d25d25b8a80 x1876699507794688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763436.284550:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507794688/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763436.284559:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763436.284563:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763436.284566:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507794688:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763436.284572:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763436.284589:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763436.284599:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffff8d25d25b9180 x1876699507794816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763436.284612:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507794816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763436.284621:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763436.284624:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763436.284627:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763436.284633:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763436.284651:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763436.284661:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763436/real 1789763436] req@ffff8d25d25bbb80 x1876699507794944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763436.284674:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507794944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763461 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763436.284683:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763436.284685:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763436.284688:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507794944:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763436.347566:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763437.371514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763438.395691:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763439.419622:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763440.443626:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763441.403569:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763441.403586:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763441.403592:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763441.403605:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3419 00000100:00080000:3.0:1789763441.403617:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763441.403627:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763441.403673:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507795072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763441.403720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763441.403725:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763441.403728:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763441.403733:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3419 00000100:00080000:3.0:1789763441.403737:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763441.403742:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763441.403754:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763441.403771:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507795200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763441.403786:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763441.403793:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507795072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763441.403794:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763441.403800:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763441.403817:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3419 00000100:00080000:3.0:1789763441.403821:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763441.403825:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763441.403834:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763441.403849:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507795328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763441.403890:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763441.403897:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763441.403902:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763441.403910:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3419 00000100:00080000:3.0:1789763441.403917:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763441.403920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763441.403929:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763441.403940:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763441.403959:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507795456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763441.403974:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763441.403979:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763441.403982:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763441.403988:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763441/real 1789763441] req@ffff8d25d1b09c00 x1876699507795072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763441.403992:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3419 00000100:00080000:3.0:1789763441.404002:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763441.404013:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763441.404026:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763441.404036:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507795072/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763441.404045:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507795584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763441.404070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763441.404074:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763441.404083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763441.404084:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763441.404092:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763441.404101:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507795072:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763441.404105:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3419 00000100:00080000:3.0:1789763441.404109:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763441.404114:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763441.404121:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763441.404137:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507795712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763441.404194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763441.404227:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763441.404246:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763441/real 1789763441] req@ffff8d25d1b0aa00 x1876699507795200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763441.404264:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507795200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763441.404285:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763441.404290:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763441.404294:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763441.404302:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763441.404322:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763441.404334:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763441/real 1789763441] req@ffff8d25d1b09180 x1876699507795328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763441.404348:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507795328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763441.404358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763441.404362:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763441.404366:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763441.404373:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795456:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763441.404455:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763441.404471:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763441/real 1789763441] req@ffff8d25d1b0ad80 x1876699507795456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763441.404486:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507795456/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763441.404496:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763441.404500:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763441.404505:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795456:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763441.404512:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507795584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763441.404532:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763441.404552:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763441/real 1789763441] req@ffff8d25d1b08a80 x1876699507795584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763441.404567:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507795584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763441.404576:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763441.404579:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763441.404583:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507795584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763441.404590:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507795712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763441.404608:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763441.404631:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763441/real 1789763441] req@ffff8d25d1b0b480 x1876699507795712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763441.404645:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507795712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763466 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763441.404654:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763441.404657:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763441.404661:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507795712:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763441.467546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763442.491597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763443.515539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763444.539588:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763445.563603:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763446.523570:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763446.523588:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763446.523596:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763446.523614:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3424 00000100:00080000:3.0:1789763446.523632:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763446.523643:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763446.523709:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507795840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763446.523756:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763446.523761:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763446.523765:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763446.523771:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3424 00000100:00080000:3.0:1789763446.523776:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763446.523781:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763446.523797:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763446.523849:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507795968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763446.523869:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763446.523874:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:1.0:1789763446.523876:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507795840:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763446.523879:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763446.523900:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3424 00000100:00080000:3.0:1789763446.523906:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763446.523913:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763446.523924:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763446.523946:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507796096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763446.523958:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763446.523963:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763446.523965:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763446.523972:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3424 00000100:00080000:3.0:1789763446.523978:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763446.523987:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:1.0:1789763446.523999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763446.524011:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763446.524047:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507796224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763446.524073:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763446.524083:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763446/real 1789763446] req@ffff8d25d1b08000 x1876699507795840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763446.524084:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763446.524094:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763446.524128:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3424 00000100:00080000:3.0:1789763446.524140:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763446.524143:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507795840/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763446.524166:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:1.0:1789763446.524173:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763446.524182:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:3.0:1789763446.524184:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763446.524192:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507795840:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763446.524223:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507796352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763446.524241:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795968:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763446.524250:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763446.524256:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763446.524260:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763446.524272:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3424 00000100:00080000:3.0:1789763446.524282:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:1.0:1789763446.524290:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763446.524302:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763446.524317:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763446.524329:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763446/real 1789763446] req@ffff8d25d1b09500 x1876699507795968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763446.524368:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507796480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763446.524372:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507795968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763446.524501:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763446.524510:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763446.524520:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507795968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763446.524535:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763446.524572:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763446.524590:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763446/real 1789763446] req@ffff8d25d1b08380 x1876699507796096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763446.524606:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507796096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763446.524624:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763446.524629:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763446.524633:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763446.524640:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796224:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763446.524660:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763446.524672:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763446/real 1789763446] req@ffff8d25d1b0b100 x1876699507796224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763446.524686:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507796224/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763446.524696:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763446.524699:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763446.524703:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796224:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763446.524730:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507796352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763446.524751:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763446.524765:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763446/real 1789763446] req@ffff8d25d1b0a300 x1876699507796352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763446.524779:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507796352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763446.524789:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763446.524792:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763446.524797:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507796352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763446.524804:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507796480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763446.524854:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763446.524871:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763446/real 1789763446] req@ffff8d25d1b08000 x1876699507796480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763446.524886:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507796480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763471 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763446.524896:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763446.524900:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763446.524904:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507796480:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763446.587625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763447.611577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763448.635626:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763449.659603:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763450.683646:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763451.643546:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763451.643564:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763451.643572:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763451.643589:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3429 00000100:00080000:2.0:1789763451.643604:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763451.643619:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763451.643683:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507796608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763451.643738:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763451.643746:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763451.643751:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763451.643761:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3429 00000100:00080000:2.0:1789763451.643810:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763451.643818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507796608:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763451.643828:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763451.643854:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763451.643889:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507796736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763451.643910:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763451.643916:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763451.643919:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763451.643927:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3429 00000100:00080000:2.0:1789763451.643937:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763451.643945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763451.643952:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763451.643963:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763451.643986:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507796864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763451.643999:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763451.644004:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763451.644008:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763451.644017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763451/real 1789763451] req@ffff8d25c8023b80 x1876699507796608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763451.644018:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3429 00000100:00080000:2.0:1789763451.644030:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763451.644043:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763451.644058:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763451.644069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507796608/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763451.644079:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507796992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763451.644109:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763451.644113:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763451.644122:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763451.644124:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763451.644133:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763451.644138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507796608:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763451.644146:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3429 00000100:00080000:2.0:1789763451.644157:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763451.644163:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763451.644168:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763451.644178:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507797120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763451.644189:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763451.644192:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763451.644194:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763451.644198:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3429 00000100:00080000:2.0:1789763451.644202:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763451.644206:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763451.644211:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763451.644220:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507797248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763451.644555:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763451.644599:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763451.644625:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763451/real 1789763451] req@ffff8d25c8022300 x1876699507796736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763451.644650:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507796736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763451.644665:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763451.644672:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763451.644681:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763451.644694:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763451.644813:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763451.644831:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763451/real 1789763451] req@ffff8d25c8020380 x1876699507796864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763451.644856:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507796864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763451.644866:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763451.644870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763451.644874:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763451.644882:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796992:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763451.644904:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763451.644916:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763451/real 1789763451] req@ffff8d25c8022d80 x1876699507796992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763451.644931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507796992/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763451.644940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763451.644944:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763451.644947:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507796992:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763451.644954:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507797120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763451.644972:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763451.644984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763451/real 1789763451] req@ffff8d25c8023100 x1876699507797120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763451.644998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507797120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763451.645016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763451.645019:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763451.645023:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507797120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763451.645029:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507797248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763451.645047:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763451.645058:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763451/real 1789763451] req@ffff8d25c8023480 x1876699507797248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763451.645071:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507797248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763476 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763451.645080:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763451.645083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763451.645087:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507797248:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763451.707624:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763452.731564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763453.755685:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763454.779671:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763455.803614:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763456.763501:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763456.763520:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763456.763528:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763456.763546:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3434 00000100:00080000:2.0:1789763456.763565:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763456.763579:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763456.763636:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507797376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763456.763677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763456.763685:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763456.763689:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763456.763699:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3434 00000100:00080000:2.0:1789763456.763709:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763456.763720:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763456.763733:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507797376:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763456.763738:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763456.763788:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507797504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763456.763807:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763456.763812:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763456.763815:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763456.763820:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3434 00000100:00080000:2.0:1789763456.763828:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763456.763834:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763456.763842:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763456.763844:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763456.763867:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507797632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763456.763883:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763456.763888:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763456.763891:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763456.763898:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3434 00000100:00080000:2.0:1789763456.763903:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763456.763911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763456/real 1789763456] req@ffff8d25c8021500 x1876699507797376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763456.763915:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763456.763929:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763456.763955:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507797376/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763456.763962:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507797760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763456.763984:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763456.763997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763456.764006:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763456.764007:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507797376:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763456.764026:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763456.764031:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763456.764041:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3434 00000100:00080000:2.0:1789763456.764052:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763456.764053:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507797504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763456.764071:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763456.764086:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763456.764095:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763456.764119:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763456/real 1789763456] req@ffff8d25c8020e00 x1876699507797504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763456.764123:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507797888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763456.764162:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507797504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763456.764205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763456.764212:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763456.764217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507797504:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763456.764222:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763456.764230:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507797632:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763456.764233:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763456.764242:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763456.764250:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3434 00000100:00080000:2.0:1789763456.764257:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763456.764263:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763456.764272:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763456.764282:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763456.764298:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507798016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763456.764316:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763456/real 1789763456] req@ffff8d25c8020700 x1876699507797632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763456.764348:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507797632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763456.764366:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763456.764371:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763456.764408:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507797632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763456.764418:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507797760:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763456.764450:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763456.764466:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763456/real 1789763456] req@ffff8d25c8021880 x1876699507797760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763456.764482:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507797760/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763456.764492:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763456.764495:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763456.764499:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507797760:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763456.764519:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507797888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763456.764540:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763456.764552:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763456/real 1789763456] req@ffff8d25c8021c00 x1876699507797888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763456.764567:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507797888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763456.764576:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763456.764579:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763456.764583:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507797888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763456.764589:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507798016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763456.764607:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763456.764619:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763456/real 1789763456] req@ffff8d25c8021500 x1876699507798016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763456.764633:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507798016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763481 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763456.764643:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763456.764647:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763456.764651:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507798016:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763456.827631:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763457.851701:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763458.875781:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763459.899671:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763460.923757:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763461.883577:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763461.883594:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763461.883601:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763461.883618:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3439 00000100:00080000:0.0:1789763461.883634:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763461.883646:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763461.883734:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507798144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763461.883810:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763461.883817:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763461.883822:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763461.883831:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3439 00000100:00080000:0.0:1789763461.883838:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763461.883846:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763461.883865:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763461.883884:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507798272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763461.883898:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763461.883903:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763461.883906:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763461.883911:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3439 00000100:00080000:0.0:1789763461.883915:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763461.883921:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507798144:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763461.883925:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763461.883937:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763461.883950:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507798400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763461.883962:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763461.883967:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763461.883969:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763461.883974:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3439 00000100:00080000:0.0:1789763461.883977:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763461.883981:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763461.883987:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763461.884001:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507798528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763461.884013:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763461.884018:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763461.884020:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763461.884024:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3439 00000100:00080000:0.0:1789763461.884027:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763461.884031:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763461.884036:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763461.884048:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507798656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763461.884058:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763461.884062:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763461.884064:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763461.884068:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3439 00000100:00080000:0.0:1789763461.884071:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763461.884075:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763461.884081:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763461.884095:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507798784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763461.884104:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507798272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763461.884178:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507798400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763461.884212:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507798528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763461.884248:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507798656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763461.884279:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507798784:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763461.947637:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763462.971600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763463.995590:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763464.955751:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789763464.955807:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763464.955834:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763464.955894:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763464.955905:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763464.955917:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763464.955933:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763464.955963:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789763464.956027:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763461/real 1789763464] req@ffff8d25d25ba680 x1876699507798400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763464.956075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507798400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763464.956091:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763464.956098:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763464.956103:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507798400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763464.956118:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763461/real 1789763464] req@ffff8d25d25bb480 x1876699507798528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763464.956134:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507798528/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763464.956143:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763464.956148:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763464.956152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507798528:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763464.956160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763461/real 1789763464] req@ffff8d25d25b9f80 x1876699507798656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763464.956174:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507798656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763464.956183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763464.956188:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763464.956195:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507798656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763464.956207:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763461/real 1789763464] req@ffff8d25d25bb800 x1876699507798784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763464.956227:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507798784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763464.956241:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763464.956248:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763464.956255:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507798784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763464.956269:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763461/real 1789763464] req@ffff8d25d25b9c00 x1876699507798144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763464.956295:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507798144/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763464.956312:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763464.956319:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763464.956327:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507798144:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763464.956339:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763461/real 1789763464] req@ffff8d25d25bb100 x1876699507798272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763464.956364:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507798272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763486 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763464.956419:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763464.956427:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763464.956437:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507798272:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763465.019594:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763466.043575:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763467.003624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763467.003640:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763467.003648:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763467.003702:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3444 00000100:00080000:2.0:1789763467.003723:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763467.003740:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763467.003813:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507798912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763467.003871:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763467.003876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763467.003879:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763467.003887:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3444 00000100:00080000:2.0:1789763467.003893:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763467.003900:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763467.003916:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763467.003937:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507799040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763467.003954:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763467.003961:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763467.003967:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763467.003969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507798912:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763467.003976:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3444 00000100:00080000:2.0:1789763467.003987:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763467.003992:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763467.004000:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763467.004011:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507799168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763467.004024:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763467.004029:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763467.004031:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763467.004037:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3444 00000100:00080000:2.0:1789763467.004042:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763467.004047:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763467.004053:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763467.004068:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507799296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763467.004083:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763467.004088:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763467.004091:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763467.004095:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3444 00000100:00080000:2.0:1789763467.004099:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763467.004109:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763467.004114:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763467.004117:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763467.004133:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507799424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763467.004153:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763467.004161:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763467.004165:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763467.004173:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3444 00000100:00080000:2.0:1789763467.004190:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763467.004199:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763467.004210:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763467.004234:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507799552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763467.004253:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffff8d25c8023480 x1876699507798912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763467.004308:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507798912/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763467.004343:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763467.004352:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763467.004361:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507798912:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763467.004454:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763467.004508:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763467.004538:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffff8d25c8023100 x1876699507799040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763467.004563:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507799040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763467.004578:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763467.004584:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763467.004591:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763467.004603:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763467.004646:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763467.004699:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffff8d25c8022d80 x1876699507799168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763467.004728:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507799168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763467.004744:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763467.004750:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763467.004757:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763467.004771:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799296:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763467.004980:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763467.005012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffff8d25c8020380 x1876699507799296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763467.005038:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507799296/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763467.005055:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763467.005061:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763467.005067:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799296:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763467.005078:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507799424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763467.005121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763467.005142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffff8d25c8022300 x1876699507799424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763467.005165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507799424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763467.005180:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763467.005187:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763467.005193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507799424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763467.005204:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507799552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763467.005245:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763467.005269:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763467/real 1789763467] req@ffff8d25c8023b80 x1876699507799552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763467.005295:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507799552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763492 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763467.005313:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763467.005319:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763467.005325:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507799552:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763467.067629:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763468.091700:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763469.115568:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763470.139564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763471.163583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763472.123542:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763472.123570:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763472.123575:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763472.123587:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3449 00000100:00080000:0.0:1789763472.123598:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763472.123607:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763472.123694:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507799680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763472.123719:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763472.123722:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763472.123724:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763472.123727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3449 00000100:00080000:0.0:1789763472.123731:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763472.123735:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763472.123745:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763472.123760:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507799680:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763472.123761:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507799808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763472.123778:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763472.123781:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763472.123782:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763472.123786:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3449 00000100:00080000:0.0:1789763472.123789:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763472.123792:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763472.123795:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763472.123805:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507799936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763472.123814:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763472.123818:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763472.123820:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763472.123822:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3449 00000100:00080000:0.0:1789763472.123825:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763472.123831:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763472.123832:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763472.123840:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763472.123851:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507800064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763472.123879:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763472.123883:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763472.123885:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763472.123890:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3449 00000100:00080000:0.0:1789763472.123895:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763472.123900:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffff8d25d25b8e00 x1876699507799680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763472.123906:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763472.123917:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763472.123944:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507800192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763472.123944:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507799680/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763472.124004:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763472.124005:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763472.124014:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763472.124020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507799680:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763472.124026:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763472.124032:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763472.124039:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3449 00000100:00080000:0.0:1789763472.124044:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763472.124048:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763472.124054:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763472.124060:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763472.124070:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507800320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763472.124109:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763472.124136:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffff8d25d25b8380 x1876699507799808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763472.124173:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507799808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763472.124191:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763472.124195:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763472.124198:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763472.124206:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763472.124231:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763472.124244:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffff8d25d25b8700 x1876699507799936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763472.124257:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507799936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763472.124265:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763472.124268:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763472.124271:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507799936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763472.124278:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800064:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763472.124295:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763472.124305:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffff8d25d25bb100 x1876699507800064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763472.124317:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507800064/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763472.124325:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763472.124328:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763472.124332:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800064:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763472.124338:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507800192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763472.124354:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763472.124364:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffff8d25d25b9c00 x1876699507800192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763472.124401:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507800192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763472.124411:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763472.124414:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763472.124418:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507800192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763472.124439:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507800320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763472.124473:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763472.124482:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763472/real 1789763472] req@ffff8d25d25bb800 x1876699507800320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763472.124491:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507800320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763497 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763472.124497:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763472.124500:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763472.124502:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507800320:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763472.187595:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763473.211605:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763474.235638:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763475.259540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763476.283553:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763477.243568:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.243584:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763477.243615:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.243634:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3454 00000100:00080000:2.0:1789763477.243650:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763477.243661:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763477.243720:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507800448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763477.243762:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.243767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763477.243770:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.243776:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3454 00000100:00080000:2.0:1789763477.243780:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763477.243786:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763477.243802:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763477.243821:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507800576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763477.243831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507800448:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763477.243848:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.243855:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763477.243857:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.243864:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3454 00000100:00080000:2.0:1789763477.243869:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763477.243874:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763477.243880:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763477.243912:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507800704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763477.243951:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.243955:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763477.243956:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.243960:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3454 00000100:00080000:2.0:1789763477.243966:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763477.243974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763477.243981:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763477.243991:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763477.244024:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507800832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763477.244039:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.244044:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763477.244046:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.244052:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3454 00000100:00080000:2.0:1789763477.244058:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763477.244071:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffff8d25c8022680 x1876699507800448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763477.244072:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763477.244092:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763477.244115:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507800448/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763477.244127:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507800960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763477.244136:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763477.244153:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763477.244168:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507800448:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763477.244170:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763477.244183:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763477.244187:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763477.244195:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3454 00000100:00080000:2.0:1789763477.244202:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763477.244209:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763477.244216:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763477.244230:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763477.244238:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507801088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763477.244314:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763477.244336:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffff8d25c8023800 x1876699507800576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763477.244372:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507800576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763477.244389:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763477.244397:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763477.244403:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763477.244415:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763477.244505:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763477.244539:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffff8d25c8020000 x1876699507800704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763477.244576:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507800704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763477.244634:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763477.244641:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763477.244647:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763477.244657:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800832:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763477.244716:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763477.244739:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffff8d25c8022a00 x1876699507800832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763477.244765:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507800832/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763477.244783:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763477.244788:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763477.244793:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507800832:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763477.244802:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507800960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763477.244833:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763477.244847:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffff8d25c8021180 x1876699507800960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763477.244862:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507800960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763477.244881:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763477.244885:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763477.244889:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507800960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763477.244929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507801088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763477.244950:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763477.244962:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763477/real 1789763477] req@ffff8d25c8020a80 x1876699507801088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763477.244975:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507801088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763502 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763477.244985:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763477.244988:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763477.245008:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507801088:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763477.307705:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763478.331555:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763479.355658:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763480.379614:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763481.403649:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763482.363636:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763482.363646:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763482.363650:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763482.363660:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3460 00000100:00080000:0.0:1789763482.363672:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763482.363679:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763482.363717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507801216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763482.363746:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763482.363750:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763482.363751:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763482.363755:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3460 00000100:00080000:0.0:1789763482.363758:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763482.363761:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763482.363771:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763482.363783:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507801344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763482.363796:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763482.363801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763482.363804:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763482.363811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3460 00000100:00100000:3.0:1789763482.363811:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507801216:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763482.363824:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763482.363835:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763482.363840:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763482.363854:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507801472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763482.363863:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763482.363866:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763482.363868:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763482.363871:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3460 00000100:00080000:0.0:1789763482.363875:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763482.363878:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763482.363882:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763482.363894:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507801600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763482.363902:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763482.363906:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763482.363908:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763482.363911:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3460 00000100:00080000:0.0:1789763482.363915:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763482.363919:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763482.363923:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763482.363928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763482.363939:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507801728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763482.363950:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763482.363954:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763482.363957:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763482.363961:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3460 00000100:00080000:0.0:1789763482.363964:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763482.363967:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763482.363972:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763482.363983:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507801856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763482.363993:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763482] req@ffff8d25d25b9f80 x1876699507801216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763482.364022:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507801216/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763482.364035:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763482.364041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763482.364046:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507801216:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763482.364079:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507801344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763482.364103:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763482.364116:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763482] req@ffff8d25d25bb480 x1876699507801344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763482.364128:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507801344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763482.364135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763482.364139:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763482.364142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507801344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763482.364147:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507801472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763482.364164:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763482.364174:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763482] req@ffff8d25d25ba680 x1876699507801472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763482.364185:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507801472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763482.364210:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763482.364214:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763482.364217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507801472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763482.364223:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507801600:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763482.364244:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763482.364272:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763482] req@ffff8d25d25bbb80 x1876699507801600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763482.364293:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507801600/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763482.364301:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763482.364304:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763482.364307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507801600:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763482.364312:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507801728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763482.364371:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763482.364424:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763482] req@ffff8d25d25b9180 x1876699507801728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763482.364446:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507801728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763482.364455:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763482.364459:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763482.364462:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507801728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763482.364469:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507801856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763482.364492:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763482.364504:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763482/real 1789763482] req@ffff8d25d25b8a80 x1876699507801856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763482.364542:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507801856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763507 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763482.364577:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763482.364595:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763482.364601:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507801856:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763482.427637:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763483.451659:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763484.475617:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763485.499663:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763486.523599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763487.483494:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763487.483509:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763487.483541:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763487.483559:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3465 00000100:00080000:0.0:1789763487.483576:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763487.483606:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763487.483664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507801984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763487.483714:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763487.483720:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763487.483723:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763487.483730:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3465 00000100:00080000:0.0:1789763487.483738:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763487.483746:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763487.483760:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763487.483796:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507801984:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763487.483798:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507802112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763487.483838:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763487.483844:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763487.483847:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763487.483853:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3465 00000100:00080000:0.0:1789763487.483862:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763487.483869:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763487.483877:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763487.483896:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763487.483901:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507802240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763487.483919:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763487.483924:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763487.483927:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763487.483933:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3465 00000100:00080000:0.0:1789763487.483939:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763487.483944:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763487.483950:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763487.483985:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507802368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763487.483985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763487/real 1789763487] req@ffff8d25d25b8000 x1876699507801984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763487.484027:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763487.484035:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763487.484038:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763487.484045:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3465 00000100:00080000:0.0:1789763487.484051:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763487.484059:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507801984/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763487.484063:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763487.484075:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763487.484093:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507802496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763487.484098:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763487.484114:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763487.484127:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763487.484134:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507801984:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763487.484135:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763487.484141:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763487.484154:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3465 00000100:00080000:0.0:1789763487.484159:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763487.484165:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763487.484171:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763487.484186:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507802624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763487.484197:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763487.484256:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763487.484283:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763487/real 1789763487] req@ffff8d25d25ba300 x1876699507802112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763487.484302:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507802112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763487.484316:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763487.484321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763487.484326:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763487.484335:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763487.484363:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763487.484437:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763487/real 1789763487] req@ffff8d25d25bad80 x1876699507802240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763487.484463:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507802240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763487.484485:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763487.484492:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763487.484497:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763487.484512:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802368:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763487.484574:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763487.484596:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763487/real 1789763487] req@ffff8d25d25b8a80 x1876699507802368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763487.484619:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507802368/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763487.484636:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763487.484643:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763487.484651:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802368:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763487.484665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507802496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763487.484703:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763487.484720:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763487/real 1789763487] req@ffff8d25d25b9180 x1876699507802496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763487.484735:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507802496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763487.484746:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763487.484749:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763487.484754:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507802496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763487.484779:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507802624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763487.484812:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763487.484825:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763487/real 1789763487] req@ffff8d25d25bbb80 x1876699507802624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763487.484839:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507802624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763512 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763487.484849:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763487.484853:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763487.484856:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507802624:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763487.547648:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763488.571617:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763489.595631:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763490.619599:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763491.643627:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763492.603570:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763492.603588:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763492.603594:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763492.603612:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3470 00000100:00080000:0.0:1789763492.603627:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763492.603639:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763492.603697:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507802752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763492.603746:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763492.603755:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763492.603759:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763492.603768:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3470 00000100:00080000:0.0:1789763492.603776:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763492.603786:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763492.603806:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763492.603833:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507802880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763492.603850:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763492.603855:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763492.603858:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763492.603866:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3470 00000100:00080000:0.0:1789763492.603873:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763492.603879:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763492.603890:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763492.603895:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507802752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763492.603915:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507803008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763492.603935:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763492.603941:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763492.603944:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763492.603951:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3470 00000100:00080000:0.0:1789763492.603956:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763492.603962:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763492.603970:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763492.603986:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507803136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763492.604002:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763492.604014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763492.604022:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763492.604027:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763492.604037:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3470 00000100:00080000:0.0:1789763492.604045:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763492.604054:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763492.604062:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763492.604082:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507803264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763492.604096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763492/real 1789763492] req@ffff8d25d25ba680 x1876699507802752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763492.604210:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763492.604219:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763492.604223:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763492.604237:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3470 00000100:00100000:2.0:1789763492.604239:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507802752/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763492.604252:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763492.604275:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763492.604283:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763492.604289:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763492.604296:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763492.604310:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507802752:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763492.604331:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763492.604332:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507803392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763492.604449:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763492.604476:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763492/real 1789763492] req@ffff8d25d25bb480 x1876699507802880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763492.604525:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507802880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763492.604537:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763492.604541:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763492.604546:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507802880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763492.604580:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763492.604607:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763492.604622:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763492/real 1789763492] req@ffff8d25d25b9f80 x1876699507803008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763492.604645:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507803008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763492.604655:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763492.604658:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763492.604662:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763492.604669:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803136:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763492.604689:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763492.604700:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763492/real 1789763492] req@ffff8d25d25bb800 x1876699507803136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763492.604714:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507803136/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763492.604724:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763492.604728:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763492.604731:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803136:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763492.604738:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507803264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763492.604757:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763492.604769:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763492/real 1789763492] req@ffff8d25d25b9c00 x1876699507803264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763492.604783:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507803264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763492.604801:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763492.604805:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763492.604808:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507803264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763492.604815:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507803392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763492.604834:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763492.604845:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763492/real 1789763492] req@ffff8d25d25bb100 x1876699507803392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763492.604858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507803392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763517 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763492.604868:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763492.604872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763492.604875:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507803392:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763492.667674:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763493.691640:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763494.715706:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763495.739633:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763496.763584:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763497.723551:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763497.723568:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763497.723575:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763497.723592:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3475 00000100:00080000:1.0:1789763497.723607:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763497.723620:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763497.723672:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507803520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763497.723710:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763497.723716:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763497.723718:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763497.723724:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3475 00000100:00080000:1.0:1789763497.723730:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763497.723736:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763497.723749:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763497.723765:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507803648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763497.723781:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763497.723785:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763497.723788:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763497.723792:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3475 00000100:00080000:1.0:1789763497.723797:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763497.723810:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763497.723817:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507803520:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789763497.723818:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763497.723859:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507803776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763497.723873:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763497.723879:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763497.723882:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763497.723888:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3475 00000100:00080000:1.0:1789763497.723892:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763497.723896:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763497.723902:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763497.723917:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507803904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763497.723929:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763497.723935:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763497.723938:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763497.723943:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3475 00000100:00080000:1.0:1789763497.723947:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763497.723960:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763497.723966:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763497.723976:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507804032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763497.723990:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763497.723994:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763497.724002:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763497.724007:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763497.724027:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3475 00000100:00080000:1.0:1789763497.724031:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763497.724037:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763497.724044:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763497.724059:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507804160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763497.724124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763497/real 1789763497] req@ffff8d25c816a680 x1876699507803520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763497.724165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507803520/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763497.724189:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763497.724199:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763497.724207:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507803520:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763497.724262:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763497.724306:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763497.724328:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763497/real 1789763497] req@ffff8d25c816aa00 x1876699507803648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763497.724346:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507803648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763497.724359:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763497.724365:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763497.724373:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763497.724401:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763497.724424:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763497.724515:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763497/real 1789763497] req@ffff8d25c8169500 x1876699507803776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763497.724543:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507803776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763497.724560:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763497.724566:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763497.724572:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763497.724584:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803904:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763497.724617:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763497.724632:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763497/real 1789763497] req@ffff8d25c8168000 x1876699507803904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763497.724648:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507803904/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763497.724663:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763497.724669:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763497.724675:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507803904:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763497.724686:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763497.724722:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763497.724743:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763497/real 1789763497] req@ffff8d25c8169f80 x1876699507804032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763497.724766:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507804032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763497.724780:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763497.724787:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763497.724794:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763497.724806:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763497.724859:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763497.724876:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763497/real 1789763497] req@ffff8d25c8168e00 x1876699507804160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763497.724902:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507804160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763522 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763497.724913:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763497.724917:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763497.724921:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804160:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763497.787609:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763498.811592:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763499.835658:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763500.859650:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763501.883662:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763502.843623:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763502.843640:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763502.843649:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763502.843666:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3480 00000100:00080000:0.0:1789763502.843695:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763502.843708:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763502.843775:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507804288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763502.843815:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763502.843820:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763502.843823:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763502.843830:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3480 00000100:00080000:0.0:1789763502.843836:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763502.843843:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763502.843858:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763502.843881:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507804416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763502.843883:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507804288:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763502.843912:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763502.843917:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763502.843919:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763502.843923:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3480 00000100:00080000:0.0:1789763502.843929:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763502.843933:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763502.843939:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763502.843952:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507804544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763502.843964:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763502.843968:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763502.843971:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763502.843977:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3480 00000100:00080000:0.0:1789763502.843982:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763502.843987:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763502.843995:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763502.844014:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507804672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763502.844021:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763502.844039:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763502.844044:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763502.844047:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763502.844054:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3480 00000100:00080000:0.0:1789763502.844069:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763502.844074:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763502.844079:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763502.844094:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507804800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763502.844106:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763502.844111:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763502.844117:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763502.844121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763502/real 1789763502] req@ffff8d25d25b8700 x1876699507804288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763502.844124:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3480 00000100:00080000:0.0:1789763502.844132:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763502.844142:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763502.844153:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763502.844172:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507804928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763502.844187:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507804288/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763502.844208:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763502.844214:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763502.844221:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507804288:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763502.844263:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507804416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763502.844299:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763502.844317:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763502/real 1789763502] req@ffff8d25d25b8380 x1876699507804416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763502.844339:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507804416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763502.844358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763502.844365:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763502.844372:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507804416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763502.844450:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507804544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763502.844485:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763502.844506:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763502/real 1789763502] req@ffff8d25d25b8e00 x1876699507804544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763502.844531:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507804544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763502.844549:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763502.844555:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763502.844562:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507804544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763502.844575:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507804672:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763502.844615:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763502.844632:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763502/real 1789763502] req@ffff8d25d25bb100 x1876699507804672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763502.844648:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507804672/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763502.844659:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763502.844663:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763502.844668:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507804672:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763502.844676:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763502.844699:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763502.844711:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763502/real 1789763502] req@ffff8d25d25b9c00 x1876699507804800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763502.844725:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507804800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763502.844735:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763502.844738:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763502.844742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763502.844748:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763502.844768:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763502.844779:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763502/real 1789763502] req@ffff8d25d25bb800 x1876699507804928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763502.844793:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507804928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763527 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763502.844803:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763502.844806:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763502.844810:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507804928:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763502.907646:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763503.931646:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763504.955667:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763505.979631:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763507.003622:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763507.963632:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763507.963648:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763507.963655:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763507.963689:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3485 00000100:00080000:2.0:1789763507.963704:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763507.963718:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763507.963782:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507805056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763507.963828:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763507.963834:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763507.963837:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763507.963843:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3485 00000100:00080000:2.0:1789763507.963847:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763507.963853:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763507.963886:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763507.963901:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507805184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763507.963914:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507805056:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763507.963915:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763507.963928:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763507.963930:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763507.963934:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3485 00000100:00080000:2.0:1789763507.963937:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763507.963941:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763507.963945:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763507.963958:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507805312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763507.963968:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763507.963972:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763507.963973:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763507.963977:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3485 00000100:00080000:2.0:1789763507.963998:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763507.964002:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763507.964007:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763507.964021:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763507.964022:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507805440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763507.964039:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763507.964045:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763507.964047:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763507.964053:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3485 00000100:00080000:2.0:1789763507.964059:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763507.964066:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763507.964091:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763507.964121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763507/real 1789763507] req@ffff8d25c8020a80 x1876699507805056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763507.964142:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507805568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763507.964160:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507805056/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763507.964164:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763507.964180:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763507.964187:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763507.964199:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763507.964200:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3485 00000100:00080000:2.0:1789763507.964214:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763507.964233:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763507.964248:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763507.964249:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507805056:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763507.964265:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763507.964291:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507805696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763507.964316:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763507.964429:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763507.964457:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763507/real 1789763507] req@ffff8d25c8021180 x1876699507805184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763507.964479:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507805184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763507.964493:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763507.964499:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763507.964503:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763507.964511:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763507.964557:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763507.964570:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763507/real 1789763507] req@ffff8d25c8022a00 x1876699507805312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763507.964584:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507805312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763507.964593:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763507.964597:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763507.964600:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763507.964622:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805440:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763507.964638:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763507.964647:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763507/real 1789763507] req@ffff8d25c8020000 x1876699507805440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763507.964664:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507805440/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763507.964691:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763507.964694:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763507.964698:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805440:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763507.964704:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507805568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763507.964726:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763507.964739:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763507/real 1789763507] req@ffff8d25c8023800 x1876699507805568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763507.964753:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507805568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763507.964764:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763507.964769:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763507.964775:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507805568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763507.964808:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507805696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763507.964851:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763507.964889:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763507/real 1789763507] req@ffff8d25c8022680 x1876699507805696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763507.964902:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507805696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763532 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763507.964915:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763507.964919:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763507.964921:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507805696:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763508.027665:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763509.051612:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763510.075701:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763511.099622:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763512.123528:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763513.083615:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.083629:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763513.083636:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.083649:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3490 00000100:00080000:0.0:1789763513.083663:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763513.083672:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763513.083721:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507805824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763513.083762:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.083767:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763513.083770:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.083775:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3490 00000100:00080000:0.0:1789763513.083780:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763513.083784:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763513.083797:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763513.083829:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507805952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763513.083864:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507805824:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763513.083886:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.083899:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763513.083903:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.083913:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3490 00000100:00080000:0.0:1789763513.083922:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763513.083931:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763513.083943:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763513.083971:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507806080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763513.083996:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.084007:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:3.0:1789763513.084016:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763513.084018:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.084050:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3490 00000100:00080000:0.0:1789763513.084059:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763513.084066:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763513.084075:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763513.084096:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507806208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763513.084125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffff8d25d25b9f80 x1876699507805824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763513.084138:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.084156:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763513.084164:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.084177:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3490 00000100:00080000:0.0:1789763513.084194:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763513.084202:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507805824/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763513.084208:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763513.084223:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763513.084244:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763513.084251:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507806336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763513.084263:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763513.084276:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507805824:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763513.084293:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763513.084300:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763513.084303:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763513.084310:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3490 00000100:00080000:0.0:1789763513.084316:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763513.084322:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763513.084351:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763513.084377:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763513.084440:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507806464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763513.084484:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763513.084508:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffff8d25d25bb480 x1876699507805952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763513.084531:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507805952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763513.084545:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763513.084551:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763513.084556:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507805952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763513.084586:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763513.084629:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763513.084654:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffff8d25d25ba680 x1876699507806080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763513.084678:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507806080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763513.084694:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763513.084701:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763513.084708:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763513.084721:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806208:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763513.084758:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763513.084774:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffff8d25d25bbb80 x1876699507806208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763513.084789:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507806208/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763513.084800:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763513.084804:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763513.084808:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806208:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763513.084815:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507806336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763513.084862:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763513.084872:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffff8d25d25b9180 x1876699507806336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763513.084882:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507806336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763513.084890:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763513.084892:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763513.084895:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507806336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763513.084913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507806464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763513.084945:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763513.084960:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763513/real 1789763513] req@ffff8d25d25b8a80 x1876699507806464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763513.084974:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507806464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763538 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763513.084984:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763513.084987:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763513.084991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507806464:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763513.147526:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763514.171740:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763515.195580:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763516.219611:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763517.243586:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763518.203552:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763518.203570:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763518.203578:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763518.203595:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3495 00000100:00080000:1.0:1789763518.203610:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763518.203623:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763518.203678:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507806592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763518.203716:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763518.203721:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763518.203735:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763518.203742:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3495 00000100:00080000:1.0:1789763518.203749:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763518.203759:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763518.203776:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763518.203804:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507806592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763518.203807:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507806720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763518.203843:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763518.203851:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763518.203855:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763518.203864:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3495 00000100:00080000:1.0:1789763518.203870:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763518.203878:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763518.203887:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763518.203904:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507806848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763518.203920:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763518.203923:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763518.203931:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763518.203934:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763518.203940:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3495 00000100:00080000:1.0:1789763518.203957:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763518.203963:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763518.203970:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763518.203986:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507806976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763518.204000:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763518.204004:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763518.204007:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763518.204018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffff8d25c8168380 x1876699507806592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763518.204030:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3495 00000100:00080000:1.0:1789763518.204037:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763518.204049:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763518.204056:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763518.204070:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507807104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763518.204085:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763518.204093:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507806592/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763518.204094:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763518.204101:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763518.204118:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3495 00000100:00080000:1.0:1789763518.204128:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763518.204129:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763518.204136:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789763518.204145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1789763518.204149:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763518.204157:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507806592:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763518.204169:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507807232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763518.204199:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763518.204240:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763518.204259:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffff8d25c8168700 x1876699507806720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763518.204275:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507806720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763518.204286:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763518.204322:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763518.204328:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763518.204337:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763518.204367:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763518.204442:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffff8d25c8168a80 x1876699507806848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763518.204471:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507806848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763518.204498:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763518.204503:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763518.204508:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763518.204516:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806976:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763518.204548:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763518.204564:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffff8d25c816a300 x1876699507806976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763518.204578:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507806976/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763518.204592:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763518.204599:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763518.204605:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507806976:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763518.204617:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507807104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763518.204654:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763518.204677:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffff8d25c816ad80 x1876699507807104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763518.204700:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507807104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763518.204711:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763518.204714:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763518.204719:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507807104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763518.204728:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507807232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763518.204755:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763518.204769:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763518/real 1789763518] req@ffff8d25c8169c00 x1876699507807232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763518.204790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507807232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763543 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763518.204805:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763518.204811:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763518.204818:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507807232:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763518.267578:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763519.291588:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763520.315556:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763521.339597:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763522.363585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763523.323563:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.323574:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763523.323582:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.323599:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3501 00000100:00080000:3.0:1789763523.323616:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.323631:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763523.323690:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507807360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763523.323734:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.323739:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763523.323742:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.323751:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3501 00000100:00080000:3.0:1789763523.323771:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.323780:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763523.323795:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763523.323797:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507807360:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763523.323824:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507807488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763523.323851:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.323858:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763523.323862:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.323870:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3501 00000100:00080000:3.0:1789763523.323880:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.323889:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763523.323901:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763523.323904:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763523.323973:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffff8d25d1b0b480 x1876699507807360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763523.324000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507807360/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763523.324015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763523.324021:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763523.324027:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507807360:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763523.324034:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507807616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763523.324054:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.324059:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763523.324062:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763523.324069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507807488:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763523.324076:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3501 00000100:00080000:3.0:1789763523.324087:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763523.324114:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763523.324132:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffff8d25d1b08a80 x1876699507807488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763523.324157:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507807488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763523.324169:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763523.324173:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763523.324177:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507807488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763523.324185:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507807616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763523.324210:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763523.324223:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffff8d25d1b0ad80 x1876699507807616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763523.324241:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507807616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763523.324286:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763523.324305:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:2.0:1789763523.324306:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763523.324321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763523.324329:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507807616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763523.324472:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507807744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763523.324517:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.324525:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763523.324530:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.324540:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3501 00000100:00080000:3.0:1789763523.324550:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.324560:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763523.324568:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507807744:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1789763523.324577:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763523.324614:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507807872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763523.324617:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763523.324643:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffff8d25d1b09180 x1876699507807744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763523.324646:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763523.324667:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763523.324672:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763523.324685:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3501 00000100:00100000:2.0:1789763523.324689:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507807744/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763523.324695:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763523.324712:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763523.324727:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763523.324731:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763523.324741:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763523.324746:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507807744:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763523.324755:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507808000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763523.324771:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507807872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763523.324802:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763523.324823:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffff8d25d1b0aa00 x1876699507807872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763523.324847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507807872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763523.324864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763523.324870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763523.324877:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507807872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763523.324902:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507808000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763523.324941:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763523.324957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763523/real 1789763523] req@ffff8d25d1b09c00 x1876699507808000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763523.324973:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507808000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763548 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763523.324983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763523.324986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763523.324991:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507808000:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763523.387541:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763524.411583:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763525.435553:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763526.459554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763527.483615:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763528.443449:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763528.443459:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763528.443465:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763528.443475:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3506 00000100:00080000:3.0:1789763528.443484:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763528.443490:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763528.443521:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507808128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763528.443543:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763528.443547:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763528.443549:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763528.443554:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3506 00000100:00080000:3.0:1789763528.443558:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763528.443562:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763528.443571:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763528.443581:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507808128:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763528.443584:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507808256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763528.443594:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763528.443597:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763528.443598:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763528.443602:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3506 00000100:00080000:3.0:1789763528.443606:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763528.443609:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763528.443613:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763528.443622:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507808384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763528.443635:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763528.443638:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763528.443640:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763528.443644:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3506 00000100:00080000:3.0:1789763528.443647:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763528.443650:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763528.443655:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763528.443664:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507808512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763528.443672:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763528.443675:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763528.443677:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763528.443678:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507808256:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763528.443681:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3506 00000100:00080000:3.0:1789763528.443684:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763528.443687:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763528.443692:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763528.443701:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507808640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763528.443711:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507808384:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763528.443711:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763528.443715:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763528.443717:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763528.443733:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3506 00000100:00080000:3.0:1789763528.443737:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763528.443741:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763528.443743:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507808512:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1789763528.443746:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763528.443759:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507808768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763528.443768:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507808640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763528.443794:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507808768:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763528.507500:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763529.531568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763530.555678:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763531.515776:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789763531.515830:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763531.515849:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763531.515896:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763531.515908:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763531.515920:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763531.515933:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763531.515948:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789763531.515970:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763528/real 1789763531] req@ffff8d25d1b0bb80 x1876699507808640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763531.516025:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507808640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763531.516046:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763531.516056:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763531.516065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507808640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763531.516080:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763528/real 1789763531] req@ffff8d25d1b0b800 x1876699507808256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763531.516105:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507808256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763531.516121:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763531.516129:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763531.516136:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507808256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763531.516148:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763528/real 1789763531] req@ffff8d25d1b09880 x1876699507808384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763531.516186:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507808384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763531.516224:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763531.516231:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763531.516237:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507808384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763531.516248:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763528/real 1789763531] req@ffff8d25d1b08e00 x1876699507808512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763531.516270:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507808512/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763531.516284:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763531.516290:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763531.516296:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507808512:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763531.516307:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763528/real 1789763531] req@ffff8d25d1b08700 x1876699507808128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763531.516330:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507808128/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763531.516346:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763531.516352:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763531.516358:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507808128:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763531.516444:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763528/real 1789763531] req@ffff8d25d1b0a680 x1876699507808768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763531.516472:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507808768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763553 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763531.516488:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763531.516497:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763531.516503:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507808768:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763531.579564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763532.603535:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763533.563626:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763533.563644:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763533.563650:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763533.563663:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3511 00000100:00080000:2.0:1789763533.563676:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763533.563685:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763533.563733:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507808896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763533.563782:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763533.563789:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763533.563793:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763533.563803:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3511 00000100:00080000:2.0:1789763533.563813:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763533.563823:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763533.563839:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763533.563872:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507809024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763533.563881:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507808896:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763533.563917:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763533.563925:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763533.563930:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763533.563943:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3511 00000100:00080000:2.0:1789763533.563951:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763533.563957:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763533.563966:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763533.563987:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507809152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763533.564000:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763533.564005:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763533.564007:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763533.564013:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3511 00000100:00080000:2.0:1789763533.564019:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763533.564023:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763533.564029:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763533.564042:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763533.564046:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507809280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763533.564062:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763533.564067:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763533.564071:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763533.564080:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3511 00000100:00080000:2.0:1789763533.564085:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763533.564090:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763533.564097:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763533.564112:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507809408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763533.564125:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763533.564139:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789763533.564142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763533/real 1789763533] req@ffff8d25c8023b80 x1876699507808896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763533.564143:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763533.564159:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3511 00000100:00080000:2.0:1789763533.564173:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763533.564223:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763533.564249:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763533.564273:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507808896/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763533.564284:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507809536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763533.564354:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763533.564373:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763533.564436:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507808896:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763533.564502:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763533.564561:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763533.564594:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763533/real 1789763533] req@ffff8d25c8022300 x1876699507809024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763533.564625:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507809024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763533.564643:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763533.564653:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763533.564660:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763533.564674:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763533.564721:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763533.564745:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763533/real 1789763533] req@ffff8d25c8020380 x1876699507809152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763533.564770:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507809152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763533.564786:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763533.564793:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763533.564799:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763533.565482:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809280:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763533.565542:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763533.565587:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763533/real 1789763533] req@ffff8d25c8022d80 x1876699507809280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763533.565611:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507809280/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763533.565627:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763533.565633:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763533.565639:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809280:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763533.565651:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507809408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763533.565692:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763533.565716:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763533/real 1789763533] req@ffff8d25c8023100 x1876699507809408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763533.565738:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507809408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763533.565755:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763533.565761:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763533.565767:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507809408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763533.565779:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507809536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763533.565832:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763533.565855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763533/real 1789763533] req@ffff8d25c8023480 x1876699507809536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763533.565879:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507809536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763558 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763533.565895:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763533.565902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763533.565909:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507809536:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763533.627616:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763534.651634:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763535.675664:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763536.699619:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763537.723597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763538.683516:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763538.683554:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763538.683563:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763538.683581:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3516 00000100:00080000:2.0:1789763538.683599:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763538.683612:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763538.683669:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507809664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763538.683706:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763538.683720:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763538.683723:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763538.683729:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3516 00000100:00080000:2.0:1789763538.683734:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763538.683740:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763538.683752:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763538.683769:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507809792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763538.683785:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763538.683790:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763538.683792:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763538.683798:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3516 00000100:00080000:2.0:1789763538.683802:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763538.683807:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763538.683813:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763538.683819:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507809664:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763538.683831:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507809920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763538.683847:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763538.683853:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763538.683855:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763538.683867:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3516 00000100:00080000:2.0:1789763538.683873:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763538.683903:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763538.683911:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763538.683918:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763538.683935:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507810048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763538.683953:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763538.683958:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763538.683961:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763538.683967:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3516 00000100:00080000:2.0:1789763538.683974:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763538.683983:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763538.683984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763538/real 1789763538] req@ffff8d25c8021500 x1876699507809664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763538.683990:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763538.684017:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507810176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763538.684030:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507809664/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763538.684039:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763538.684053:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763538.684060:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763538.684069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763538.684071:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3516 00000100:00080000:2.0:1789763538.684083:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763538.684085:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763538.684093:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763538.684098:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507809664:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763538.684103:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763538.684115:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763538.684126:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507810304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763538.684198:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763538.684220:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763538/real 1789763538] req@ffff8d25c8021c00 x1876699507809792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763538.684237:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507809792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763538.684248:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763538.684253:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763538.684257:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763538.684287:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763538.684311:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763538.684324:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763538/real 1789763538] req@ffff8d25c8021880 x1876699507809920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763538.684339:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507809920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763538.684357:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763538.684361:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763538.684365:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507809920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763538.684371:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810048:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763538.684439:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763538.684453:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763538/real 1789763538] req@ffff8d25c8020700 x1876699507810048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763538.684468:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507810048/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763538.684478:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763538.684482:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763538.684486:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810048:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763538.684493:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507810176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763538.684512:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763538.684524:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763538/real 1789763538] req@ffff8d25c8020e00 x1876699507810176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763538.684538:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507810176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763538.684548:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763538.684551:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763538.684556:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507810176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763538.684562:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507810304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763538.684580:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763538.684591:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763538/real 1789763538] req@ffff8d25c8023480 x1876699507810304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763538.684605:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507810304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763563 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763538.684619:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763538.684625:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763538.684631:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507810304:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763538.747644:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763539.771586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763540.795645:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763541.819643:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763542.843581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763543.803532:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763543.803545:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763543.803550:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763543.803563:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3521 00000100:00080000:2.0:1789763543.803580:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763543.803593:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763543.803656:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507810432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763543.803709:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763543.803715:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763543.803717:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763543.803724:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3521 00000100:00080000:2.0:1789763543.803730:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763543.803737:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763543.803750:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763543.803771:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507810560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763543.803788:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763543.803798:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789763543.803800:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507810432:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763543.803805:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763543.803826:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3521 00000100:00080000:2.0:1789763543.803832:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763543.803837:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763543.803843:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763543.803859:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507810688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763543.803872:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763543.803877:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763543.803880:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763543.803885:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3521 00000100:00080000:2.0:1789763543.803912:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763543.803917:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763543.803924:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763543.803938:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507810816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763543.803954:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763543.803972:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763543.803977:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763543.803980:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763543.803986:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3521 00000100:00080000:2.0:1789763543.803991:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763543.803997:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763543.804003:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763543.804020:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507810944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763543.804033:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763543.804039:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763543.804041:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763543.804046:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3521 00000100:00080000:2.0:1789763543.804052:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763543.804060:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763543/real 1789763543] req@ffff8d25c8023100 x1876699507810432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763543.804063:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763543.804082:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763543.804159:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507810432/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763543.804160:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507811072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763543.804289:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763543.804298:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763543.804307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507810432:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763543.804351:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763543.804429:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763543.804450:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763543/real 1789763543] req@ffff8d25c8022d80 x1876699507810560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763543.804467:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507810560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763543.804478:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763543.804482:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763543.804486:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763543.804494:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763543.804518:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763543.804530:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763543/real 1789763543] req@ffff8d25c8020380 x1876699507810688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763543.804544:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507810688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763543.804554:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763543.804557:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763543.804560:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763543.804568:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810816:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763543.804588:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763543.804599:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763543/real 1789763543] req@ffff8d25c8022300 x1876699507810816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763543.804624:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507810816/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763543.804640:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763543.804646:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763543.804652:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507810816:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763543.804664:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507810944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763543.804703:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763543.804726:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763543/real 1789763543] req@ffff8d25c8023b80 x1876699507810944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763543.804744:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507810944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763543.804754:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763543.804758:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763543.804762:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507810944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763543.804769:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507811072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763543.804795:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763543.804807:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763543/real 1789763543] req@ffff8d25c8022680 x1876699507811072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763543.804820:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507811072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763568 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763543.804838:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763543.804842:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763543.804845:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507811072:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763543.867680:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763544.891615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763545.915589:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763546.939562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763547.963542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763548.923478:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763548.923491:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763548.923497:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763548.923510:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3526 00000100:00080000:2.0:1789763548.923523:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763548.923531:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763548.923568:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507811200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763548.923604:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763548.923608:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763548.923609:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763548.923614:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3526 00000100:00080000:2.0:1789763548.923620:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763548.923628:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763548.923641:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763548.923642:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507811200:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763548.923668:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507811328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763548.923705:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763548.923710:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763548.923713:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763548.923720:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3526 00000100:00080000:2.0:1789763548.923738:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763548.923761:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763548.923768:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763548.923774:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763548.923794:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507811456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763548.923805:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763548.923808:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763548.923812:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763548.923814:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763548/real 1789763548] req@ffff8d25c8023800 x1876699507811200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763548.923820:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3526 00000100:00080000:2.0:1789763548.923829:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763548.923837:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763548.923844:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763548.923847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507811200/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763548.923858:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507811584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763548.923901:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763548.923902:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763548.923909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763548.923910:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763548.923915:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763548.923918:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507811200:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763548.923924:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3526 00000100:00080000:2.0:1789763548.923929:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763548.923933:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763548.923938:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763548.923947:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507811712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763548.923951:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507811328:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763548.923972:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763548.923976:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763548.923978:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763548.923982:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3526 00000100:00080000:2.0:1789763548.923987:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763548.923993:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763548.923995:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763548.923999:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763548.924012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763548/real 1789763548] req@ffff8d25c8020000 x1876699507811328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763548.924015:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507811840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763548.924050:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507811328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763548.924061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763548.924083:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763548.924088:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507811328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763548.924094:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507811456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763548.924115:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763548.924127:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763548/real 1789763548] req@ffff8d25c8022a00 x1876699507811456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763548.924137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507811456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763548.924143:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763548.924146:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763548.924148:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507811456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763548.924153:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507811584:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763548.924165:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763548.924172:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763548/real 1789763548] req@ffff8d25c8021180 x1876699507811584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763548.924181:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507811584/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763548.924188:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763548.924190:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763548.924192:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507811584:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763548.924204:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507811712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763548.924217:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763548.924225:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763548/real 1789763548] req@ffff8d25c8020a80 x1876699507811712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763548.924237:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507811712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763548.924247:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763548.924251:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763548.924256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507811712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763548.924263:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507811840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763548.924289:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763548.924322:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763548/real 1789763548] req@ffff8d25c8023800 x1876699507811840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763548.924335:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507811840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763573 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763548.924353:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763548.924357:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763548.924361:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507811840:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763548.987519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763550.011602:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763551.035522:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763552.059614:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763553.083659:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763554.043505:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763554.043518:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763554.043523:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763554.043549:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3531 00000100:00080000:2.0:1789763554.043566:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763554.043579:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763554.043655:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507811968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763554.043693:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763554.043698:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763554.043701:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763554.043706:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3531 00000100:00080000:2.0:1789763554.043712:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763554.043717:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763554.043731:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763554.043748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507812096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763554.043764:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763554.043770:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789763554.043770:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507811968:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763554.043777:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763554.043787:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3531 00000100:00080000:2.0:1789763554.043791:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763554.043797:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763554.043804:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763554.043820:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507812224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763554.043831:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763554.043855:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763554.043859:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763554.043864:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3531 00000100:00080000:2.0:1789763554.043883:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763554.043888:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763554.043896:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763554.043903:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763554.043921:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507812352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763554.043937:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763554.043942:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763554.043944:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763554.043950:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3531 00000100:00080000:2.0:1789763554.043957:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763554.043962:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763554.043978:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763554.043980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffff8d25c8022680 x1876699507811968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763554.043999:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507812480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763554.044023:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763554.044077:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763554.044091:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763554.044105:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3531 00000100:00080000:2.0:1789763554.044111:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763554.044116:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763554.044124:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507811968/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763554.044155:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763554.044179:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763554.044184:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507812608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763554.044209:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763554.044225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507811968:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763554.044271:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763554.044325:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763554.044344:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffff8d25c8023b80 x1876699507812096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763554.044378:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507812096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763554.044389:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763554.044393:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763554.044397:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763554.044403:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763554.044466:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763554.044480:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffff8d25c8022300 x1876699507812224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763554.044495:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507812224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763554.044525:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763554.044539:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763554.044543:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763554.044550:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812352:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763554.044573:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763554.044585:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffff8d25c8020380 x1876699507812352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763554.044598:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507812352/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763554.044616:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763554.044620:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763554.044624:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812352:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763554.044631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507812480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763554.044651:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763554.044664:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffff8d25c8022d80 x1876699507812480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763554.044677:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507812480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763554.044686:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763554.044690:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763554.044694:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507812480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763554.044700:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507812608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763554.044718:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763554.044729:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763554/real 1789763554] req@ffff8d25c8023100 x1876699507812608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763554.044741:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507812608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763579 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763554.044750:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763554.044753:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763554.044757:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507812608:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763554.107578:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763555.131595:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763556.155817:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763557.179512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763558.203653:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763559.163542:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.163555:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763559.163561:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763559.163575:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3536 00000100:00080000:0.0:1789763559.163587:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.163600:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763559.163659:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507812736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763559.163701:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.163707:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763559.163711:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763559.163717:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3536 00000100:00080000:0.0:1789763559.163721:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.163727:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763559.163740:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763559.163761:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507812864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763559.163778:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507812736:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763559.163794:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.163800:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763559.163803:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763559.163812:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3536 00000100:00080000:0.0:1789763559.163819:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.163826:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763559.163835:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763559.163857:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507812992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763559.163871:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.163879:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1789763559.163880:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763559.163884:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763559.163893:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3536 00000100:00080000:0.0:1789763559.163901:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.163906:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763559.163914:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763559.163934:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507813120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763559.163949:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763559.163954:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763559.163963:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763559.163965:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffff8d25d25bad80 x1876699507812736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763559.163971:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3536 00000100:00080000:0.0:1789763559.163979:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.164014:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763559.164027:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763559.164063:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507813248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763559.164064:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507812736/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763559.164114:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763559.164119:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763559.164122:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763559.164130:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763559.164134:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763559.164142:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3536 00000100:00100000:2.0:1789763559.164152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507812736:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763559.164153:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763559.164166:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763559.164174:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763559.164186:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507813376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763559.164201:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763559.164247:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763559.164267:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffff8d25d25ba300 x1876699507812864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.164290:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507812864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.164301:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.164306:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763559.164312:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763559.164320:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763559.164344:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763559.164361:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffff8d25d25b8000 x1876699507812992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.164428:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507812992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.164448:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.164457:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763559.164467:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507812992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763559.164487:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813120:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763559.164542:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763559.164568:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffff8d25d25b8a80 x1876699507813120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.164586:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507813120/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.164598:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.164603:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763559.164607:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813120:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763559.164615:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507813248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763559.164638:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763559.164651:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffff8d25d25b9180 x1876699507813248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.164673:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507813248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.164689:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.164695:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763559.164702:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507813248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763559.164738:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507813376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763559.164776:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763559.164793:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763559/real 1789763559] req@ffff8d25d25bbb80 x1876699507813376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763559.164809:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507813376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763584 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763559.164819:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763559.164823:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763559.164827:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507813376:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763559.227619:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763560.251602:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763561.275554:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763562.299680:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763563.323587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763564.283584:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763564.283601:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763564.283609:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763564.283627:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3541 00000100:00080000:3.0:1789763564.283643:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763564.283654:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763564.283704:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507813504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763564.283740:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763564.283745:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763564.283748:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763564.283754:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3541 00000100:00080000:3.0:1789763564.283759:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763564.283764:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763564.283777:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763564.283793:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507813632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763564.283810:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763564.283821:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507813504:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763564.283821:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763564.283829:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763564.283843:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3541 00000100:00080000:3.0:1789763564.283847:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763564.283852:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763564.283859:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763564.283874:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507813760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763564.283889:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763564.283894:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763564.283896:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763564.283901:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3541 00000100:00080000:3.0:1789763564.283907:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763564.283911:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763564.283917:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763564.283921:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763564.283940:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507813888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763564.283984:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763564.283989:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763564.283991:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763564.283997:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3541 00000100:00080000:3.0:1789763564.284004:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763564.284010:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763564.284019:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763564.284022:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffff8d25d1b0a680 x1876699507813504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763564.284042:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507814016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763564.284085:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763564.284090:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763564.284093:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763564.284098:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507813504/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763564.284105:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3541 00000100:00080000:3.0:1789763564.284116:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763564.284126:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763564.284131:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763564.284133:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763564.284143:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763564.284152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507813504:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763564.284158:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507814144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763564.284194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763564.284232:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763564.284250:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffff8d25d1b08700 x1876699507813632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763564.284266:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507813632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763564.284277:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763564.284281:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763564.284285:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763564.284292:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763564.284313:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763564.284325:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffff8d25d1b08e00 x1876699507813760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763564.284339:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507813760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763564.284357:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763564.284361:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763564.284365:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763564.284372:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813888:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763564.284439:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763564.284455:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffff8d25d1b09880 x1876699507813888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763564.284474:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507813888/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763564.284492:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763564.284498:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763564.284522:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507813888:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763564.284536:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763564.284578:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763564.284613:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffff8d25d1b0b800 x1876699507814016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763564.284630:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507814016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763564.284640:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763564.284644:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763564.284649:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763564.284657:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763564.284683:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763564.284696:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763564/real 1789763564] req@ffff8d25d1b0bb80 x1876699507814144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763564.284710:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507814144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763589 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763564.284720:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763564.284723:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763564.284726:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814144:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763564.347601:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763565.371575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763566.395571:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763567.419621:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763568.443569:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763569.403527:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763569.403544:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763569.403552:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763569.403569:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3547 00000100:00080000:3.0:1789763569.403584:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763569.403596:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763569.403653:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507814272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763569.403705:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763569.403712:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763569.403716:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763569.403724:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3547 00000100:00080000:3.0:1789763569.403732:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763569.403740:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763569.403757:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763569.403787:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507814400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763569.403793:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507814272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763569.403828:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763569.403836:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763569.403840:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763569.403851:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3547 00000100:00080000:3.0:1789763569.403875:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763569.403882:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789763569.403892:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763569.403892:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763569.403940:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507814528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763569.403973:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763569.403978:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763569.403981:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763569.403990:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3547 00000100:00100000:2.0:1789763569.403991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763569/real 1789763569] req@ffff8d25d1b09c00 x1876699507814272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763569.403997:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763569.404011:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763569.404026:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763569.404039:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507814272/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763569.404050:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507814656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763569.404063:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763569.404074:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763569.404079:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763569.404084:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763569.404089:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763569.404091:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507814272:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763569.404099:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3547 00000100:00080000:3.0:1789763569.404106:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763569.404112:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763569.404118:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763569.404130:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507814784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763569.404139:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507814400:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763569.404154:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763569.404162:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763569.404165:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763569.404171:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3547 00000100:00080000:3.0:1789763569.404190:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763569.404199:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763569.404201:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763569.404215:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763569.404233:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507814912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763569.404245:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763569/real 1789763569] req@ffff8d25d1b0aa00 x1876699507814400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763569.404282:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507814400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763569.404301:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763569.404306:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763569.404311:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507814400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763569.404320:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507814528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763569.404352:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763569.404369:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763569/real 1789763569] req@ffff8d25d1b09180 x1876699507814528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763569.404419:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507814528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763569.404430:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763569.404434:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763569.404438:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507814528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763569.404445:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507814656:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763569.404470:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763569.404484:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763569/real 1789763569] req@ffff8d25d1b0ad80 x1876699507814656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763569.404508:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507814656/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763569.404518:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763569.404522:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763569.404526:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507814656:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763569.404550:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763569.404570:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763569.404583:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763569/real 1789763569] req@ffff8d25d1b08a80 x1876699507814784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763569.404598:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507814784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763569.404608:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763569.404611:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763569.404615:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763569.404622:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763569.404640:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763569.404652:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763569/real 1789763569] req@ffff8d25d1b0b480 x1876699507814912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763569.404666:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507814912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763594 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763569.404689:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763569.404695:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763569.404702:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507814912:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763569.467564:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763570.491611:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763571.515523:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763572.539653:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763573.563546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763574.523641:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763574.523658:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763574.523664:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763574.523681:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3552 00000100:00080000:0.0:1789763574.523699:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763574.523713:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763574.523782:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507815040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763574.523829:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763574.523835:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763574.523837:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763574.523843:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3552 00000100:00080000:0.0:1789763574.523847:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763574.523853:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763574.523871:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763574.523912:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507815168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763574.523928:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763574.523935:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763574.523940:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763574.523943:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507815040:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763574.523948:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3552 00000100:00080000:0.0:1789763574.523957:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763574.523967:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763574.523974:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763574.523992:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507815296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763574.524005:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763574.524010:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763574.524012:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763574.524018:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3552 00000100:00080000:0.0:1789763574.524022:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763574.524026:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763574.524033:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763574.524048:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507815424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763574.524062:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763574.524067:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763574.524070:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763574.524074:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3552 00000100:00080000:0.0:1789763574.524078:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763574.524082:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763574.524088:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763574.524098:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763574.524109:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507815552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763574.524122:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763574.524127:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763574.524129:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763574.524136:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3552 00000100:00080000:0.0:1789763574.524153:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763574.524158:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763574.524165:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763574.524182:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507815680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763574.524217:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763574/real 1789763574] req@ffff8d25d25ba680 x1876699507815040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763574.524265:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507815040/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763574.524288:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763574.524297:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763574.524307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507815040:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763574.524357:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763574.524456:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763574.524480:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763574/real 1789763574] req@ffff8d25d25bb480 x1876699507815168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763574.524498:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507815168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763574.524509:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763574.524515:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763574.524520:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763574.524528:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763574.524574:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763574.524590:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763574/real 1789763574] req@ffff8d25d25b9f80 x1876699507815296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763574.524606:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507815296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763574.524617:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763574.524632:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763574.524636:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763574.524644:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815424:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763574.524665:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763574.524677:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763574/real 1789763574] req@ffff8d25d25bb800 x1876699507815424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763574.524691:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507815424/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763574.524702:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763574.524706:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763574.524710:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815424:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763574.524717:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507815552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763574.524737:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763574.524750:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763574/real 1789763574] req@ffff8d25d25b9c00 x1876699507815552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763574.524764:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507815552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763574.524774:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763574.524777:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763574.524781:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507815552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763574.524787:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507815680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763574.524806:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763574.524818:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763574/real 1789763574] req@ffff8d25d25bb100 x1876699507815680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763574.524842:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507815680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763599 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763574.524852:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763574.524855:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763574.524859:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507815680:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763574.587687:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763575.611554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763576.635601:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763577.659575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763578.683647:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763579.643489:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763579.643517:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763579.643524:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763579.643542:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3557 00000100:00080000:0.0:1789763579.643558:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763579.643571:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763579.643653:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507815808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763579.643709:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763579.643715:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763579.643718:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763579.643726:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3557 00000100:00080000:0.0:1789763579.643733:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763579.643740:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763579.643758:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763579.643766:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507815808:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763579.643795:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507815936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763579.643821:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763579.643830:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763579.643835:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763579.643865:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3557 00000100:00080000:0.0:1789763579.643875:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763579.643883:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763579.643903:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763579.643924:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507816064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763579.643932:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763579.643950:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763579.643966:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763579.643969:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763579.643978:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3557 00000100:00080000:0.0:1789763579.643986:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763579.643991:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763579.643998:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763579.644015:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507816192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763579.644030:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763579/real 1789763579] req@ffff8d25d25b8e00 x1876699507815808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763579.644034:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763579.644049:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763579.644056:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763579.644071:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3557 00000100:00080000:0.0:1789763579.644078:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763579.644085:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763579.644091:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507815808/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763579.644095:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763579.644121:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763579.644123:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507816320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763579.644137:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763579.644152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507815808:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763579.644170:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763579.644180:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763579.644185:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763579.644196:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3557 00000100:00100000:2.0:1789763579.644203:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763579.644208:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763579.644223:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763579.644236:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763579.644249:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763579.644263:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507816448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763579.644284:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763579/real 1789763579] req@ffff8d25d25b8380 x1876699507815936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763579.644307:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507815936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763579.644319:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763579.644326:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763579.644332:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507815936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763579.644341:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763579.644374:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763579.644461:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763579/real 1789763579] req@ffff8d25d25b8700 x1876699507816064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763579.644485:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507816064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763579.644505:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763579.644510:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763579.644514:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763579.644523:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816192:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763579.644555:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763579.644570:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763579/real 1789763579] req@ffff8d25d25bb100 x1876699507816192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763579.644585:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507816192/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763579.644605:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763579.644608:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763579.644611:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816192:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763579.644618:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507816320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763579.644639:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763579.644651:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763579/real 1789763579] req@ffff8d25d25b9c00 x1876699507816320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763579.644666:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507816320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763579.644676:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763579.644679:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763579.644684:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507816320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763579.644710:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507816448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763579.644730:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763579.644743:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763579/real 1789763579] req@ffff8d25d25bb800 x1876699507816448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763579.644758:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507816448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763604 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763579.644768:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763579.644771:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763579.644775:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507816448:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763579.707674:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763580.731674:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763581.755611:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763582.779615:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763583.803655:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763584.763528:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763584.763540:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763584.763546:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763584.763559:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3562 00000100:00080000:3.0:1789763584.763572:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763584.763581:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763584.763629:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507816576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763584.763666:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763584.763671:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763584.763673:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763584.763679:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3562 00000100:00080000:3.0:1789763584.763684:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763584.763691:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763584.763704:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763584.763727:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507816704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763584.763729:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507816576:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763584.763759:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763584.763763:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763584.763765:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763584.763770:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3562 00000100:00080000:3.0:1789763584.763775:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763584.763780:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763584.763785:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763584.763828:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507816832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763584.763843:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763584.763848:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763584.763850:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763584.763856:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3562 00000100:00080000:3.0:1789763584.763863:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763584.763867:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763584.763875:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763584.763883:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763584.763899:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507816960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763584.763915:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763584.763920:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763584.763923:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763584.763929:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3562 00000100:00080000:3.0:1789763584.763934:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763584.763939:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763584.763945:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763584.763957:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507817088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763584.763970:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763584.763979:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763584.763981:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763584/real 1789763584] req@ffff8d25d1b08000 x1876699507816576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763584.763983:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763584.763998:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3562 00000100:00080000:3.0:1789763584.764008:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763584.764021:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763584.764029:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763584.764049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507816576/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763584.764050:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507817216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763584.764120:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763584.764131:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763584.764142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507816576:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763584.764184:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763584.764235:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763584.764262:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763584/real 1789763584] req@ffff8d25d1b0a300 x1876699507816704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763584.764289:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507816704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763584.764305:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763584.764312:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763584.764319:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763584.764331:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763584.764368:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763584.764432:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763584/real 1789763584] req@ffff8d25d1b0b100 x1876699507816832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763584.764456:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507816832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763584.764471:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763584.764477:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763584.764484:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763584.764495:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816960:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763584.764532:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763584.764553:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763584/real 1789763584] req@ffff8d25d1b08380 x1876699507816960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763584.764588:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507816960/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763584.764604:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763584.764610:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763584.764617:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507816960:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763584.764628:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763584.764662:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763584.764683:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763584/real 1789763584] req@ffff8d25d1b09500 x1876699507817088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763584.764708:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507817088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763584.764724:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763584.764730:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763584.764736:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763584.764747:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763584.764780:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763584.764824:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763584/real 1789763584] req@ffff8d25d1b0b480 x1876699507817216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763584.764849:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507817216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763609 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763584.764878:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763584.764885:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763584.764892:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817216:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763584.827597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763585.851631:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763586.875651:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763587.899632:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763588.923519:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763589.883476:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763589.883486:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763589.883500:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763589.883513:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3567 00000100:00080000:3.0:1789763589.883526:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763589.883534:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763589.883572:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507817344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763589.883600:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763589.883603:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763589.883604:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763589.883625:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3567 00000100:00080000:3.0:1789763589.883629:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763589.883632:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763589.883642:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763589.883655:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507817472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763589.883666:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763589.883672:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763589.883691:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507817344:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763589.883692:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763589.883704:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3567 00000100:00080000:3.0:1789763589.883709:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763589.883713:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763589.883718:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763589.883734:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507817600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763589.883746:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763589.883750:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763589.883752:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763589.883756:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3567 00000100:00080000:3.0:1789763589.883780:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763589.883802:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763589.883807:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763589.883813:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763589.883825:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507817728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763589.883838:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763589.883842:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763589.883844:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763589.883849:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3567 00000100:00100000:2.0:1789763589.883853:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763589/real 1789763589] req@ffff8d25d1b08a80 x1876699507817344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763589.883853:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763589.883859:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763589.883869:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763589.883878:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507817344/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763589.883882:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507817856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763589.883898:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763589.883900:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763589.883905:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763589.883906:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763589.883910:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763589.883912:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507817344:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763589.883916:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3567 00000100:00080000:3.0:1789763589.883921:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763589.883924:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763589.883927:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763589.883933:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507817984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763589.883939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507817472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763589.883967:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763589.883979:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763589/real 1789763589] req@ffff8d25d1b0ad80 x1876699507817472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763589.883988:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507817472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763589.883994:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763589.883996:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763589.883999:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507817472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763589.884003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507817600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763589.884014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763589.884020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763589/real 1789763589] req@ffff8d25d1b09180 x1876699507817600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763589.884028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507817600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763589.884033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763589.884035:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763589.884038:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507817600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763589.884044:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507817728:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763589.884080:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763589.884095:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763589/real 1789763589] req@ffff8d25d1b0aa00 x1876699507817728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763589.884112:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507817728/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763589.884122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763589.884125:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763589.884128:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507817728:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763589.884133:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763589.884152:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763589.884162:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763589/real 1789763589] req@ffff8d25d1b09c00 x1876699507817856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763589.884171:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507817856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763589.884178:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763589.884180:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763589.884183:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763589.884200:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763589.884214:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763589.884222:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763589/real 1789763589] req@ffff8d25d1b0bb80 x1876699507817984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763589.884236:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507817984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763614 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763589.884246:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763589.884250:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763589.884254:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507817984:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763589.947672:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763590.971687:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763591.995591:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763593.019548:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763594.043540:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763595.003674:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763595.003693:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763595.003702:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763595.003723:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3572 00000100:00080000:2.0:1789763595.003767:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763595.003779:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763595.003838:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507818112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763595.003880:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763595.003885:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763595.003889:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763595.003895:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3572 00000100:00080000:2.0:1789763595.003902:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763595.003910:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763595.003925:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763595.003950:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507818240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763595.003973:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763595.003977:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507818112:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763595.003981:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763595.003985:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763595.004003:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3572 00000100:00080000:2.0:1789763595.004009:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763595.004015:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763595.004022:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763595.004036:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507818368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763595.004057:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763595.004061:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763595.004065:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763595.004070:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3572 00000100:00080000:2.0:1789763595.004074:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763595.004078:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763595.004087:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763595.004098:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507818496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763595.004115:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763595.004121:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763595.004125:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763595.004132:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3572 00000100:00080000:2.0:1789763595.004141:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763595.004148:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763595.004157:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763595.004179:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507818624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763595.004201:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763595.004207:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763595.004212:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763595.004224:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3572 00000100:00080000:2.0:1789763595.004235:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763595.004248:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763595.004261:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763595.004281:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507818240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763595.004297:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507818752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763595.004350:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507818368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763595.004455:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507818496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763595.004521:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507818624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763595.004604:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507818752:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763595.067599:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763596.091630:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763597.115626:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763598.075815:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789763598.075901:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763598.075922:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763598.075979:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763598.075998:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763598.076014:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763598.076032:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789763598.076039:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763598] req@ffff8d25c8020e00 x1876699507818240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1789763598.076046:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789763598.076084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507818240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763598.076107:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763598.076114:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763598.076120:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507818240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763598.076131:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763598] req@ffff8d25c8020700 x1876699507818368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763598.076158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507818368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763598.076169:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763598.076174:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763598.076193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507818368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763598.076202:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763598] req@ffff8d25c8021880 x1876699507818496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763598.076218:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507818496/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763598.076228:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763598.076233:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763598.076237:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507818496:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763598.076246:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763598] req@ffff8d25c8021c00 x1876699507818624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763598.076270:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507818624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763598.076286:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763598.076293:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763598.076300:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507818624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763598.076312:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763598] req@ffff8d25c8023480 x1876699507818112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763598.076339:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507818112/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763598.076353:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763598.076360:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763598.076367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507818112:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763598.076486:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763595/real 1789763598] req@ffff8d25c8021500 x1876699507818752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763598.076516:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507818752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763620 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763598.076533:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763598.076541:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763598.076548:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507818752:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763598.139586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763599.163643:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763600.123552:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763600.123566:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763600.123572:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763600.123587:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3577 00000100:00080000:2.0:1789763600.123599:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763600.123622:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763600.123668:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507818880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763600.123740:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763600.123749:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763600.123752:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763600.123761:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3577 00000100:00080000:2.0:1789763600.123769:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763600.123777:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763600.123793:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763600.123856:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507818880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763600.123856:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507819008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763600.123894:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763600.123901:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763600.123905:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763600.123914:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3577 00000100:00080000:2.0:1789763600.123924:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763600.123952:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763600.123965:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763600.123999:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763600.124000:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507819136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763600.124025:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763600.124033:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763600.124038:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763600.124046:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3577 00000100:00080000:2.0:1789763600.124055:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763600.124062:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763600.124074:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763600.124085:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffff8d25c8021500 x1876699507818880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763600.124106:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507819264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763600.124136:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763600.124147:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507818880/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763600.124147:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763600.124157:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763600.124176:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3577 00000100:00080000:2.0:1789763600.124188:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763600.124198:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763600.124201:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763600.124217:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789763600.124220:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763600.124235:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507818880:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763600.124255:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507819392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763600.124281:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763600.124289:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819008:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763600.124291:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763600.124300:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763600.124314:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3577 00000100:00080000:2.0:1789763600.124324:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763600.124333:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763600.124344:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763600.124362:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763600.124417:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507819520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763600.124525:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffff8d25c8023480 x1876699507819008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763600.124555:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507819008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763600.124572:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763600.124579:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763600.124585:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763600.124598:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763600.124642:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763600.124670:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffff8d25c8021c00 x1876699507819136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763600.124730:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507819136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763600.124759:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763600.124765:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763600.124770:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763600.124779:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819264:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763600.124826:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763600.124845:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffff8d25c8021880 x1876699507819264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763600.124863:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507819264/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763600.124876:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763600.124880:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763600.124885:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819264:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763600.124908:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507819392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763600.124938:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763600.124957:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffff8d25c8020700 x1876699507819392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763600.124978:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507819392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763600.124987:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763600.124990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763600.124993:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507819392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763600.125001:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507819520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763600.125031:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763600.125048:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763600/real 1789763600] req@ffff8d25c8020e00 x1876699507819520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763600.125066:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507819520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763625 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763600.125079:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763600.125082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763600.125085:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507819520:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763600.187542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763601.211572:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763602.235571:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763603.259582:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763604.283566:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763605.243490:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763605.243504:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763605.243510:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763605.243523:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3582 00000100:00080000:2.0:1789763605.243536:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763605.243546:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763605.243608:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507819648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763605.243644:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763605.243672:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763605.243679:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763605.243687:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3582 00000100:00080000:2.0:1789763605.243694:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763605.243700:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763605.243713:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763605.243717:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507819648:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763605.243738:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507819776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763605.243757:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763605.243762:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763605.243766:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763605.243771:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3582 00000100:00080000:2.0:1789763605.243777:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763605.243781:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763605.243789:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763605.243808:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763605.243810:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507819904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763605.243828:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763605.243833:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763605.243837:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763605.243843:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3582 00000100:00080000:2.0:1789763605.243847:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763605.243851:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763605.243858:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763605.243878:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffff8d25c8023100 x1876699507819648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763605.243880:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507820032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763605.243933:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763605.243940:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763605.243944:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763605.243954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507819648/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763605.243957:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3582 00000100:00080000:2.0:1789763605.243973:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763605.243980:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763605.243988:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763605.243994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763605.244004:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507819648:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763605.244008:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763605.244042:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763605.244044:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507820160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763605.244077:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763605.244082:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:0.0:1789763605.244089:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763605.244090:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763605.244104:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3582 00000100:00080000:2.0:1789763605.244113:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763605.244118:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763605.244125:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763605.244142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffff8d25c8022d80 x1876699507819776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763605.244147:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507820288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763605.244230:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507819776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.244244:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.244249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763605.244254:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763605.244263:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763605.244295:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763605.244312:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffff8d25c8020380 x1876699507819904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763605.244327:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507819904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.244337:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.244341:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763605.244345:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507819904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763605.244352:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820032:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763605.244372:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763605.244418:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffff8d25c8022300 x1876699507820032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763605.244434:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507820032/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.244444:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.244448:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763605.244452:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820032:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763605.244474:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507820160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763605.244498:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763605.244513:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffff8d25c8023b80 x1876699507820160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763605.244527:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507820160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.244537:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.244540:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763605.244544:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507820160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763605.244551:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507820288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763605.244569:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763605.244581:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763605/real 1789763605] req@ffff8d25c8022680 x1876699507820288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763605.244595:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507820288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763630 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763605.244605:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763605.244608:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763605.244612:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507820288:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763605.307597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763606.331617:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763607.355613:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763608.379593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763609.403692:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763610.363596:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763610.363635:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763610.363642:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763610.363655:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3588 00000100:00080000:0.0:1789763610.363679:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763610.363688:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763610.363732:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507820416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763610.363772:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763610.363777:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763610.363780:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763610.363785:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3588 00000100:00080000:0.0:1789763610.363789:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763610.363794:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763610.363805:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763610.363823:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507820544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763610.363839:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763610.363846:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763610.363850:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507820416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763610.363851:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763610.363869:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3588 00000100:00080000:0.0:1789763610.363876:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763610.363881:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763610.363887:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763610.363905:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507820672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763610.363918:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763610.363923:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763610.363926:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763610.363931:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3588 00000100:00080000:0.0:1789763610.363937:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763610.363941:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763610.363949:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763610.363951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763610.363974:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507820800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763610.364001:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763610.364005:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763610.364007:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763610.364014:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3588 00000100:00080000:0.0:1789763610.364019:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763610.364025:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763610.364033:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffff8d25d25b9f80 x1876699507820416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763610.364036:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763610.364063:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507820928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763610.364076:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507820416/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763610.364084:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763610.364105:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763610.364114:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763610.364123:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763610.364130:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3588 00000100:00080000:2.0:1789763610.364147:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763610.364151:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763610.364162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507820416:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763610.364170:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763610.364187:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763610.364212:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507821056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763610.364223:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763610.364286:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763610.364309:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffff8d25d25bb480 x1876699507820544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763610.364327:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507820544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763610.364339:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763610.364343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763610.364367:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763610.364419:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763610.364466:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763610.364485:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffff8d25d25ba680 x1876699507820672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763610.364510:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507820672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763610.364524:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763610.364528:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763610.364532:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763610.364540:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820800:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763610.364563:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763610.364576:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffff8d25d25bbb80 x1876699507820800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763610.364590:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507820800/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763610.364600:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763610.364604:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763610.364607:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507820800:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763610.364641:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507820928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763610.364670:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763610.364686:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffff8d25d25b9180 x1876699507820928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763610.364701:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507820928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763610.364719:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763610.364723:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763610.364728:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507820928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763610.364752:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507821056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763610.364772:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763610.364785:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763610/real 1789763610] req@ffff8d25d25b8a80 x1876699507821056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763610.364799:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507821056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763635 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763610.364809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763610.364812:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763610.364816:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507821056:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763610.427551:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763611.451661:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763612.475547:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763613.499692:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763614.523663:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763615.483648:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763615.483666:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763615.483673:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763615.483689:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3593 00000100:00080000:2.0:1789763615.483707:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763615.483718:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763615.483782:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507821184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763615.483833:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763615.483839:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763615.483842:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763615.483848:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3593 00000100:00080000:2.0:1789763615.483853:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763615.483859:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763615.483876:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763615.483895:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507821312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763615.483913:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507821184:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763615.483920:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763615.483936:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763615.483938:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763615.483946:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3593 00000100:00080000:2.0:1789763615.483952:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763615.483960:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763615.483967:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763615.483987:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507821440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763615.483999:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763615.484005:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763615.484007:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763615.484013:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3593 00000100:00080000:2.0:1789763615.484018:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763615.484023:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763615.484031:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763615.484048:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507821568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763615.484063:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763615.484076:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763615.484083:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763615.484087:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763615.484095:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3593 00000100:00080000:2.0:1789763615.484102:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763615.484107:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763615.484113:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763615.484126:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507821696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763615.484142:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763615.484148:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763615.484150:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763615.484156:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3593 00000100:00080000:2.0:1789763615.484159:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763615.484164:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763615.484171:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763615.484176:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763615/real 1789763615] req@ffff8d25c8023800 x1876699507821184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763615.484195:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507821824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763615.484218:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507821184/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763615.484237:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763615.484248:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763615.484255:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507821184:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763615.484288:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507821312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763615.484326:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763615.484345:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763615/real 1789763615] req@ffff8d25c8020a80 x1876699507821312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763615.484362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507821312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763615.484373:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763615.484413:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763615.484418:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507821312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763615.484431:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507821440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763615.484460:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763615.484476:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763615/real 1789763615] req@ffff8d25c8021180 x1876699507821440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763615.484492:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507821440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763615.484502:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763615.484507:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763615.484511:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507821440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763615.484518:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507821568:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763615.484539:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763615.484551:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763615/real 1789763615] req@ffff8d25c8022a00 x1876699507821568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763615.484566:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507821568/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763615.484610:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763615.484617:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763615.484624:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507821568:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763615.484638:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507821696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763615.484681:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763615.484707:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763615/real 1789763615] req@ffff8d25c8020000 x1876699507821696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763615.484730:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507821696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763615.484743:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763615.484749:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763615.484759:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507821696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763615.484771:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507821824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763615.484804:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763615.484819:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763615/real 1789763615] req@ffff8d25c8022680 x1876699507821824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763615.484835:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507821824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763640 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763615.484847:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763615.484851:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763615.484855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507821824:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763615.547629:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763616.571665:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763617.595679:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763618.619626:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763619.643602:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763620.603660:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763620.603676:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763620.603683:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763620.603698:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3598 00000100:00080000:3.0:1789763620.603712:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763620.603721:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763620.603768:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507821952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763620.603810:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763620.603815:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763620.603818:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763620.603823:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3598 00000100:00080000:3.0:1789763620.603828:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763620.603832:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763620.603844:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763620.603859:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507822080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763620.603872:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763620.603876:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763620.603878:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763620.603882:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3598 00000100:00080000:3.0:1789763620.603885:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763620.603890:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763620.603896:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763620.603909:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507822208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763620.603924:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763620.603929:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763620.603931:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763620.603941:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507821952:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763620.603942:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3598 00000100:00080000:3.0:1789763620.603953:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763620.603962:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763620.603968:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763620.603984:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507822336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763620.603997:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763620.604002:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763620.604004:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763620.604010:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3598 00000100:00080000:3.0:1789763620.604015:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763620.604019:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763620.604026:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763620.604043:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763620.604057:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507822464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763620.604073:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763620.604077:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763620.604080:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763620.604085:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3598 00000100:00080000:3.0:1789763620.604090:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763620.604097:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763620.604109:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763620.604126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763620/real 1789763620] req@ffff8d25d1b0b800 x1876699507821952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763620.604138:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507822592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763620.604191:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507821952/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763620.604218:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763620.604224:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763620.604230:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507821952:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763620.604242:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763620.604275:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763620.604294:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763620/real 1789763620] req@ffff8d25d1b09880 x1876699507822080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763620.604311:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507822080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763620.604321:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763620.604325:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763620.604330:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763620.604363:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763620.604435:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763620.604451:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763620/real 1789763620] req@ffff8d25d1b08e00 x1876699507822208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763620.604466:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507822208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763620.604476:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763620.604480:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763620.604484:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763620.604492:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822336:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763620.604513:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763620.604525:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763620/real 1789763620] req@ffff8d25d1b08700 x1876699507822336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763620.604564:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507822336/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763620.604575:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763620.604579:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763620.604583:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822336:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763620.604591:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507822464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763620.604614:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763620.604627:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763620/real 1789763620] req@ffff8d25d1b0a680 x1876699507822464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763620.604642:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507822464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763620.604651:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763620.604655:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763620.604658:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507822464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763620.604665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507822592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763620.604684:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763620.604696:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763620/real 1789763620] req@ffff8d25d1b0bb80 x1876699507822592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763620.604711:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507822592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763645 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763620.604721:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763620.604724:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763620.604728:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507822592:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763620.667621:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763621.691615:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763622.715598:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763623.739604:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763624.763661:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763625.723533:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763625.723546:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763625.723552:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763625.723565:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3603 00000100:00080000:3.0:1789763625.723577:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763625.723598:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763625.723645:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507822720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763625.723687:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763625.723692:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763625.723694:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763625.723700:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3603 00000100:00080000:3.0:1789763625.723704:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763625.723710:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763625.723723:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763625.723739:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507822848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763625.723751:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507822720:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763625.723758:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763625.723766:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763625.723769:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763625.723775:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3603 00000100:00080000:3.0:1789763625.723779:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763625.723783:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763625.723789:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763625.723806:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507822976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763625.723818:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763625.723822:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763625.723825:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763625.723830:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3603 00000100:00080000:3.0:1789763625.723836:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763625.723843:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763625.723846:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763625.723856:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763625.723871:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507823104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763625.723884:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763625.723889:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763625.723891:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763625.723897:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3603 00000100:00080000:3.0:1789763625.723902:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763625.723912:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763625.723914:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763625/real 1789763625] req@ffff8d25d1b09c00 x1876699507822720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763625.723918:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763625.723939:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507823232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763625.723951:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763625.723958:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763625.723959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507822720/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763625.723961:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763625.723974:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3603 00000100:00080000:3.0:1789763625.723982:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763625.723991:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763625.723999:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763625.724011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:3.0:1789763625.724019:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507823360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763625.724030:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763625.724050:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507822720:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763625.724098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763625.724136:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763625.724155:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763625/real 1789763625] req@ffff8d25d1b0aa00 x1876699507822848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763625.724172:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507822848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763625.724184:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763625.724189:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763625.724193:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763625.724200:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763625.724221:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763625.724234:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763625/real 1789763625] req@ffff8d25d1b09180 x1876699507822976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763625.724248:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507822976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763625.724258:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763625.724261:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763625.724265:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507822976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763625.724272:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823104:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763625.724291:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763625.724303:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763625/real 1789763625] req@ffff8d25d1b0ad80 x1876699507823104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763625.724317:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507823104/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763625.724326:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763625.724330:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763625.724334:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823104:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763625.724341:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507823232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763625.724360:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763625.724372:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763625/real 1789763625] req@ffff8d25d1b08a80 x1876699507823232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763625.724420:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507823232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763625.724430:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763625.724434:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763625.724438:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507823232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763625.724445:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507823360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763625.724468:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763625.724482:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763625/real 1789763625] req@ffff8d25d1b0b480 x1876699507823360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763625.724497:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507823360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763650 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763625.724535:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763625.724539:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763625.724543:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507823360:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763625.787664:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763626.811623:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763627.835592:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763628.859690:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763629.883655:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763630.843552:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763630.843574:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763630.843584:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763630.843627:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3608 00000100:00080000:3.0:1789763630.843648:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763630.843665:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763630.843728:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507823488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763630.843772:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763630.843779:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763630.843782:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763630.843789:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3608 00000100:00080000:3.0:1789763630.843799:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763630.843806:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763630.843827:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763630.843834:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507823488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763630.843864:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507823616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763630.843895:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763630.843904:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763630.843909:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763630.843919:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3608 00000100:00080000:3.0:1789763630.843931:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763630.843939:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763630.843947:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763630.843967:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507823744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763630.843981:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763630.843990:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763630.843996:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763630.843999:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763630.844006:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3608 00000100:00080000:3.0:1789763630.844011:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763630.844017:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763630.844023:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763630.844034:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507823872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763630.844048:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763630.844052:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763630.844054:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763630.844058:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3608 00000100:00100000:2.0:1789763630.844094:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763630/real 1789763630] req@ffff8d25d1b09500 x1876699507823488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763630.844136:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507823488/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763630.844160:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763630.844163:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763630.844172:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763630.844179:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763630.844190:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763630.844194:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507823488:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763630.844212:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507824000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763630.844237:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763630.844242:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763630.844247:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763630.844260:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823616:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763630.844261:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3608 00000100:00080000:3.0:1789763630.844275:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763630.844287:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763630.844299:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763630.844342:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763630.844367:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763630/real 1789763630] req@ffff8d25d1b08380 x1876699507823616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763630.844435:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507823616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763630.844450:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763630.844456:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763630.844491:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763630.844508:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763630.844568:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763630.844602:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763630/real 1789763630] req@ffff8d25d1b0b100 x1876699507823744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763630.844620:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507824128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763630.844634:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507823744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763630.844658:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763630.844666:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763630.844673:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763630.844685:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823872:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763630.844720:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763630.844741:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763630/real 1789763630] req@ffff8d25d1b0a300 x1876699507823872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763630.844762:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507823872/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763630.844774:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763630.844781:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763630.844787:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507823872:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763630.844800:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763630.844830:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763630.844858:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763630/real 1789763630] req@ffff8d25d1b08000 x1876699507824000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763630.844897:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507824000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763630.844916:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763630.844923:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763630.844930:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763630.844974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763630.845020:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763630.845046:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763630/real 1789763630] req@ffff8d25d1b09500 x1876699507824128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763630.845067:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507824128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763655 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763630.845078:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763630.845082:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763630.845086:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824128:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763630.907678:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763631.931598:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763632.955620:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763633.979640:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763635.003600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763635.963574:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763635.963588:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763635.963593:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763635.963607:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3613 00000100:00080000:3.0:1789763635.963619:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763635.963628:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763635.963673:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507824256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763635.963711:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763635.963716:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763635.963719:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763635.963725:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3613 00000100:00080000:3.0:1789763635.963730:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763635.963736:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763635.963748:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763635.963765:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507824384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763635.963781:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763635.963788:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763635.963789:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507824256:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763635.963791:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763635.963805:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3613 00000100:00080000:3.0:1789763635.963813:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763635.963818:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763635.963823:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763635.963839:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507824512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763635.963865:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763635.963874:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763635.963877:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763635.963882:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3613 00000100:00080000:3.0:1789763635.963889:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763635.963894:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763635.963900:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763635.963911:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763635.963920:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507824640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763635.963946:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763635.963950:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763635.963953:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763635.963959:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3613 00000100:00080000:3.0:1789763635.963965:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763635.963971:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763635.963977:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763635.963980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763635/real 1789763635] req@ffff8d25d1b0b480 x1876699507824256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763635.963996:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507824768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763635.964015:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763635.964020:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763635.964026:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763635.964028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507824256/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763635.964043:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3613 00000100:00080000:3.0:1789763635.964058:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763635.964068:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763635.964073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763635.964075:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763635.964092:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507824896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763635.964116:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763635.964129:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507824256:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763635.964164:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507824384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763635.964201:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763635.964221:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763635/real 1789763635] req@ffff8d25d1b08a80 x1876699507824384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763635.964239:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507824384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763635.964250:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763635.964255:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763635.964259:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507824384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763635.964266:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507824512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763635.964288:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763635.964301:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763635/real 1789763635] req@ffff8d25d1b0ad80 x1876699507824512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763635.964315:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507824512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763635.964325:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763635.964343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763635.964351:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507824512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763635.964361:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507824640:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763635.964415:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763635.964469:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763635/real 1789763635] req@ffff8d25d1b09180 x1876699507824640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763635.964495:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507824640/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763635.964513:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763635.964520:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763635.964527:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507824640:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763635.964539:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763635.964581:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763635.964599:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763635/real 1789763635] req@ffff8d25d1b0aa00 x1876699507824768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763635.964614:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507824768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763635.964625:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763635.964628:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763635.964632:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763635.964640:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763635.964674:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763635.964698:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763635/real 1789763635] req@ffff8d25d1b09c00 x1876699507824896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763635.964724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507824896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763660 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763635.964741:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763635.964748:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763635.964755:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507824896:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763636.027584:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763637.051692:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763638.075664:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763639.099583:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763640.123625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763641.083564:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763641.083585:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763641.083594:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763641.083614:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3618 00000100:00080000:3.0:1789763641.083634:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763641.083650:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763641.083722:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507825024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763641.083768:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763641.083773:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763641.083776:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763641.083783:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3618 00000100:00080000:3.0:1789763641.083789:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763641.083795:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763641.083812:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763641.083833:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507825152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763641.083859:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507825024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763641.083861:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763641.083873:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763641.083878:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763641.083889:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3618 00000100:00080000:3.0:1789763641.083899:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763641.083908:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763641.083920:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763641.083948:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507825280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763641.083963:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763641.083970:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763641.083973:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763641.083978:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3618 00000100:00080000:3.0:1789763641.083989:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763641.083995:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763641.084003:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763641.084014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763641.084037:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507825408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763641.084055:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763641.084063:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763641.084067:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763641.084077:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3618 00000100:00080000:3.0:1789763641.084087:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763641.084088:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffff8d25d1b0bb80 x1876699507825024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763641.084104:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763641.084126:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763641.084152:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507825024/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763641.084162:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507825536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763641.084211:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763641.084214:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763641.084226:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763641.084232:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763641.084233:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763641.084252:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3618 00000100:00100000:2.0:1789763641.084258:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507825024:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763641.084261:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763641.084277:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763641.084289:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763641.084308:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507825664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763641.084328:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763641.084433:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763641.084461:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffff8d25d1b0a680 x1876699507825152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763641.084480:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507825152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763641.084494:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763641.084499:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763641.084504:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763641.084512:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763641.084543:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763641.084557:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffff8d25d1b08700 x1876699507825280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763641.084573:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507825280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763641.084584:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763641.084588:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763641.084592:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763641.084599:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825408:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763641.084621:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763641.084632:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffff8d25d1b08e00 x1876699507825408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763641.084647:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507825408/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763641.084658:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763641.084661:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763641.084666:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825408:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763641.084674:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507825536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763641.084694:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763641.084706:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffff8d25d1b09880 x1876699507825536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763641.084720:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507825536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763641.084730:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763641.084734:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763641.084737:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507825536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763641.084776:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507825664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763641.084814:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763641.084838:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763641/real 1789763641] req@ffff8d25d1b0b800 x1876699507825664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763641.084864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507825664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763666 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763641.084879:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763641.084883:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763641.084887:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507825664:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763641.147602:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763642.171610:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763643.195655:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763644.219638:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763645.243606:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763646.203559:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763646.203577:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763646.203584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763646.203601:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3623 00000100:00080000:1.0:1789763646.203618:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763646.203629:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763646.203690:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507825792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763646.203763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763646.203773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763646.203777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763646.203788:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3623 00000100:00080000:1.0:1789763646.203798:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763646.203808:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763646.203828:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507825792:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789763646.203829:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763646.203861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507825920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763646.203879:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763646.203885:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763646.203887:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763646.203893:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3623 00000100:00080000:1.0:1789763646.203899:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763646.203905:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763646.203912:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763646.203931:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507826048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763646.203944:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763646.203964:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763646.203967:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763646.203972:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3623 00000100:00080000:1.0:1789763646.203978:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763646.203989:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763646.203990:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763646.204005:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763646.204021:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507826176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763646.204036:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763646.204041:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763646.204055:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763646.204061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3623 00000100:00080000:1.0:1789763646.204065:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763646.204071:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763646.204077:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763646.204098:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507826304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763646.204113:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffff8d25c816b480 x1876699507825792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763646.204131:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763646.204143:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763646.204147:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763646.204158:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507825792/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763646.204161:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3623 00000100:00080000:1.0:1789763646.204176:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763646.204188:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763646.204191:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763646.204203:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:1.0:1789763646.204211:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763646.204213:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507825792:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763646.204244:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507826432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763646.204270:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763646.204314:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763646.204335:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffff8d25c8169880 x1876699507825920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763646.204415:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507825920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763646.204432:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763646.204439:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763646.204446:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507825920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763646.204457:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763646.204491:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763646.204516:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffff8d25c8169c00 x1876699507826048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763646.204556:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507826048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763646.204574:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763646.204582:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763646.204589:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763646.204602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826176:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763646.204636:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763646.204653:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffff8d25c816ad80 x1876699507826176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763646.204670:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507826176/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763646.204681:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763646.204685:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763646.204689:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826176:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763646.204695:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507826304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763646.204716:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763646.204728:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffff8d25c816a300 x1876699507826304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763646.204743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507826304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763646.204762:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763646.204766:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763646.204770:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507826304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763646.204794:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507826432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763646.204816:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763646.204829:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763646/real 1789763646] req@ffff8d25c8168a80 x1876699507826432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763646.204843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507826432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763671 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763646.204854:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763646.204858:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763646.204862:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507826432:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763646.267648:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763647.291614:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763648.315643:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763649.339527:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763650.363634:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763651.323556:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763651.323568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763651.323574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763651.323587:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3629 00000100:00080000:3.0:1789763651.323599:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763651.323609:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763651.323662:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507826560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763651.323715:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763651.323723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763651.323727:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763651.323736:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3629 00000100:00080000:3.0:1789763651.323746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763651.323754:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763651.323768:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763651.323786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507826560:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763651.323806:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507826688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763651.323847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763651.323857:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763651.323861:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763651.323886:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3629 00000100:00080000:3.0:1789763651.323892:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763651.323902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763651.323909:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763651.323922:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763651.323943:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507826816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763651.323974:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffff8d25d1b0b800 x1876699507826560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763651.323993:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763651.324004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763651.324009:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763651.324012:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507826560/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763651.324025:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3629 00000100:00080000:2.0:1789763651.324037:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763651.324047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763651.324074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507826560:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763651.324078:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763651.324088:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763651.324095:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763651.324114:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507826944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763651.324115:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826688:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763651.324138:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763651.324142:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763651.324145:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763651.324149:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3629 00000100:00080000:3.0:1789763651.324164:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763651.324187:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763651.324195:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763651.324205:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763651.324222:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffff8d25d1b09880 x1876699507826688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763651.324228:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507827072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763651.324281:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507826688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763651.324286:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763651.324301:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763651.324308:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763651.324356:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3629 00000100:00080000:2.0:1789763651.324358:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763651.324362:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763651.324368:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763651.324374:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763651.324410:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826688:192.168.203.141@tcp:38:kworker.0 00000008:00100000:3.0:1789763651.324411:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763651.324434:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507827200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763651.324440:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763651.324479:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763651.324494:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffff8d25d1b08e00 x1876699507826816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763651.324507:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507826816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763651.324515:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763651.324518:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763651.324522:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763651.324541:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826944:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763651.324559:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763651.324569:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffff8d25d1b08700 x1876699507826944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763651.324580:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507826944/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763651.324588:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763651.324591:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763651.324594:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507826944:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763651.324600:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763651.324614:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763651.324623:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffff8d25d1b0a680 x1876699507827072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763651.324634:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507827072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763651.324642:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763651.324644:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763651.324647:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763651.324652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763651.324666:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763651.324675:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763651/real 1789763651] req@ffff8d25d1b0bb80 x1876699507827200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763651.324693:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507827200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763676 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763651.324700:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763651.324703:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763651.324706:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827200:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763651.387598:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763652.411608:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763653.435621:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763654.459736:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763655.483636:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763656.443538:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763656.443549:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763656.443553:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763656.443563:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3634 00000100:00080000:0.0:1789763656.443573:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763656.443581:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763656.443636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507827328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763656.443699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763656.443704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763656.443706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763656.443711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3634 00000100:00080000:0.0:1789763656.443716:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763656.443721:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763656.443733:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763656.443772:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507827456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763656.443796:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763656.443805:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507827328:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763656.443806:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763656.443813:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763656.443832:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3634 00000100:00080000:0.0:1789763656.443838:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763656.443843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763656.443850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763656.443866:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507827584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763656.443877:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763656.443880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763656.443883:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763656.443887:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3634 00000100:00080000:0.0:1789763656.443892:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763656.443896:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763656.443900:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763656.443912:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507827712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763656.443923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763656.443926:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763656.443928:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763656.443940:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763656.443941:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3634 00000100:00080000:0.0:1789763656.443956:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763656.443961:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763656.443968:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763656.443982:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507827840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763656.443991:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763656.443995:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763656.443996:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763656.444000:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3634 00000100:00080000:0.0:1789763656.444003:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763656.444017:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763656.444023:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763656.444031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763656/real 1789763656] req@ffff8d25d25b8000 x1876699507827328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763656.444059:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507827968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763656.444082:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507827328/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763656.444103:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763656.444109:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763656.444127:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507827328:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763656.444166:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507827456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763656.444197:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763656.444214:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763656/real 1789763656] req@ffff8d25d25ba300 x1876699507827456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763656.444230:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507827456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763656.444242:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763656.444258:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763656.444262:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507827456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763656.444269:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507827584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763656.444316:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763656.444341:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763656/real 1789763656] req@ffff8d25d25bad80 x1876699507827584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763656.444356:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507827584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763656.444366:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763656.444370:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763656.444374:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507827584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763656.444441:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507827712:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763656.444467:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763656.444482:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763656/real 1789763656] req@ffff8d25d25b8a80 x1876699507827712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763656.444497:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507827712/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763656.444507:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763656.444511:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763656.444516:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507827712:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763656.444524:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763656.444556:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763656.444568:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763656/real 1789763656] req@ffff8d25d25b9180 x1876699507827840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763656.444582:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507827840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763656.444592:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763656.444596:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763656.444599:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763656.444615:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763656.444635:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763656.444647:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763656/real 1789763656] req@ffff8d25d25bbb80 x1876699507827968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763656.444661:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507827968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763681 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763656.444671:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763656.444675:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763656.444678:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507827968:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763656.507618:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763657.531573:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763658.555676:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763659.579587:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763660.603653:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763661.563569:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763661.563583:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763661.563590:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763661.563605:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3639 00000100:00080000:0.0:1789763661.563619:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763661.563629:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763661.563678:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507828096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763661.563718:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763661.563723:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763661.563726:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763661.563732:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3639 00000100:00080000:0.0:1789763661.563737:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763661.563743:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763661.563756:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763661.563774:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507828224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763661.563797:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763661.563801:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507828096:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763661.563803:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763661.563807:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763661.563825:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3639 00000100:00080000:0.0:1789763661.563833:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763661.563838:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763661.563843:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763661.563860:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507828352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763661.563872:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763661.563876:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763661.563878:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763661.563883:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3639 00000100:00080000:0.0:1789763661.563887:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763661.563892:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763661.563897:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763661.563910:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507828480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763661.563922:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763661.563926:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763661.563928:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763661.563932:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3639 00000100:00080000:0.0:1789763661.563937:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763661.563941:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763661.563947:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763661.563976:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507828608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763661.563988:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763661.563993:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763661.563996:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763661.564001:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3639 00000100:00080000:0.0:1789763661.564008:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763661.564016:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763661.564028:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763661.564048:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507828736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763661.564054:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763661.564137:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763661.564169:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763661.564194:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507828608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763661.564225:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507828736:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763661.627685:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763662.651593:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763663.675576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763664.635623:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789763664.635673:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763664.635692:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763664.635737:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763664.635754:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763664.635767:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763664.635783:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763664.635801:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789763664.635805:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763661/real 1789763664] req@ffff8d25d25bb480 x1876699507828224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763664.635859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507828224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763664.635879:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763664.635887:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763664.635893:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763664.635906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763661/real 1789763664] req@ffff8d25d25b9f80 x1876699507828352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763664.635926:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507828352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763664.635939:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763664.635944:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763664.635956:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763664.635967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763661/real 1789763664] req@ffff8d25d25bb800 x1876699507828480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763664.635987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507828480/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763664.635999:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763664.636004:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763664.636008:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828480:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763664.636019:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763661/real 1789763664] req@ffff8d25d25b9c00 x1876699507828608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763664.636049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507828608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763664.636061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763664.636066:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763664.636071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507828608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763664.636081:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763661/real 1789763664] req@ffff8d25d25ba680 x1876699507828096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763664.636127:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507828096/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763664.636140:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763664.636146:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763664.636167:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507828096:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763664.636246:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763661/real 1789763664] req@ffff8d25d25bb100 x1876699507828736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763664.636269:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507828736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763686 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763664.636282:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763664.636288:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763664.636292:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507828736:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763664.699602:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763665.723615:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763666.683491:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763666.683502:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763666.683507:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763666.683518:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3644 00000100:00080000:0.0:1789763666.683532:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763666.683541:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763666.683583:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507828864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763666.683617:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763666.683622:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763666.683624:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763666.683630:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3644 00000100:00080000:0.0:1789763666.683636:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763666.683641:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763666.683651:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763666.683660:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507828864:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763666.683673:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507828992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763666.683686:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763666.683691:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763666.683694:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763666.683699:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3644 00000100:00080000:0.0:1789763666.683704:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763666.683708:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763666.683712:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763666.683723:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507829120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763666.683740:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763666.683745:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763666.683747:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763666.683752:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3644 00000100:00080000:0.0:1789763666.683758:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763666.683764:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763666.683771:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763666.683779:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763666.683828:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507829248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763666.683844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763666/real 1789763666] req@ffff8d25d25b8700 x1876699507828864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763666.683845:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763666.683860:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763666.683866:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763666.683875:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3644 00000100:00080000:0.0:1789763666.683882:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763666.683886:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507828864/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763666.683893:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763666.683923:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763666.683931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763666.683942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763666.683950:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507829376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763666.683955:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507828864:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763666.683982:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763666.683988:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763666.683993:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763666.684002:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3644 00000100:00080000:0.0:1789763666.684012:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763666.684016:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828992:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763666.684027:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763666.684042:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763666.684073:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763666.684090:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507829504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763666.684103:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763666/real 1789763666] req@ffff8d25d25b8380 x1876699507828992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763666.684135:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507828992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763666.684153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763666.684163:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763666.684172:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507828992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763666.684224:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763666.684268:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763666.684294:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763666/real 1789763666] req@ffff8d25d25b8e00 x1876699507829120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763666.684321:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507829120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763666.684335:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763666.684343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763666.684348:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763666.684361:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829248:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763666.684461:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763666.684490:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763666/real 1789763666] req@ffff8d25d25bb100 x1876699507829248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763666.684511:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507829248/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763666.684523:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763666.684528:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763666.684532:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829248:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763666.684542:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507829376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763666.684571:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763666.684589:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763666/real 1789763666] req@ffff8d25d25ba680 x1876699507829376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763666.684606:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507829376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763666.684617:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763666.684622:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763666.684626:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507829376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763666.684656:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507829504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763666.684939:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763666.684984:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763666/real 1789763666] req@ffff8d25d25b9c00 x1876699507829504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763666.685021:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507829504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763691 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763666.685036:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763666.685041:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763666.685047:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507829504:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763666.747564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763667.771714:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763668.795547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763669.819647:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763670.843667:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763671.803593:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763671.803607:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763671.803613:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763671.803628:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3649 00000100:00080000:3.0:1789763671.803643:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763671.803654:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763671.803701:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507829632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763671.803739:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763671.803744:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763671.803747:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763671.803753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3649 00000100:00080000:3.0:1789763671.803758:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763671.803764:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763671.803776:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763671.803794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507829760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763671.803818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507829632:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763671.803822:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763671.803842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763671.803846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763671.803854:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3649 00000100:00080000:3.0:1789763671.803857:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763671.803862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763671.803869:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763671.803892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507829888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763671.803905:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763671.803910:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763671.803914:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763671.803918:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3649 00000100:00080000:3.0:1789763671.803942:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763671.803949:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763671.803956:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763671.803967:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763671.803974:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507830016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763671.803992:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763671.803997:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763671.803999:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763671.804006:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3649 00000100:00080000:3.0:1789763671.804012:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763671.804018:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763671.804027:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763671.804039:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507830144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763671.804053:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763671.804058:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763671.804062:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763671.804069:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763671/real 1789763671] req@ffff8d25d1b09c00 x1876699507829632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763671.804075:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3649 00000100:00080000:3.0:1789763671.804089:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763671.804109:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763671.804130:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763671.804145:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507829632/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763671.804156:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507830272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763671.804194:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763671.804208:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763671.804217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507829632:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763671.804270:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763671.804304:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763671.804322:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763671/real 1789763671] req@ffff8d25d1b0aa00 x1876699507829760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763671.804340:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507829760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763671.804352:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763671.804356:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763671.804361:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763671.804368:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763671.804449:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763671.804466:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763671/real 1789763671] req@ffff8d25d1b09180 x1876699507829888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763671.804482:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507829888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763671.804492:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763671.804496:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763671.804500:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507829888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763671.804508:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830016:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763671.804531:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763671.804543:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763671/real 1789763671] req@ffff8d25d1b0ad80 x1876699507830016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763671.804557:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507830016/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763671.804577:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763671.804581:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763671.804584:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830016:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763671.804591:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507830144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763671.804611:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763671.804623:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763671/real 1789763671] req@ffff8d25d1b08a80 x1876699507830144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763671.804636:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507830144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763671.804646:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763671.804650:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763671.804654:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507830144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763671.804661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507830272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763671.804681:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763671.804692:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763671/real 1789763671] req@ffff8d25d1b0b480 x1876699507830272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763671.804705:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507830272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763696 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763671.804715:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763671.804719:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763671.804724:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507830272:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763671.867584:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763672.891643:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763673.915644:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763674.939584:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763675.963593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763676.923538:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763676.923552:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763676.923558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763676.923574:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3654 00000100:00080000:2.0:1789763676.923587:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763676.923598:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763676.923650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507830400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763676.923699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763676.923707:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763676.923711:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763676.923721:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3654 00000100:00080000:2.0:1789763676.923730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763676.923740:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763676.923757:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763676.923766:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507830400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763676.923783:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507830528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763676.923800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763676.923805:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763676.923807:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763676.923812:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3654 00000100:00080000:2.0:1789763676.923817:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763676.923825:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763676.923831:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763676.923845:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507830656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763676.923859:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763676.923863:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763676.923865:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763676.923869:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3654 00000100:00080000:2.0:1789763676.923874:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763676.923882:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763676.923886:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763676.923895:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763676.923910:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507830784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763676.923923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763676.923927:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763676.923930:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763676.923935:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3654 00000100:00080000:2.0:1789763676.923938:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763676.923942:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763676.923949:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763676.923963:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763676/real 1789763676] req@ffff8d25c8023b80 x1876699507830400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763676.923967:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507830912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763676.924027:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763676.924033:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763676.924036:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763676.924045:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507830400/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763676.924046:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3654 00000100:00080000:2.0:1789763676.924056:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763676.924064:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763676.924075:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763676.924095:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507831040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763676.924096:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763676.924121:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763676.924153:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507830400:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763676.924195:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763676.924245:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763676.924278:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763676/real 1789763676] req@ffff8d25c8022300 x1876699507830528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763676.924304:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507830528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763676.924316:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763676.924321:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763676.924326:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763676.924335:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763676.924363:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763676.924410:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763676/real 1789763676] req@ffff8d25c8020380 x1876699507830656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763676.924427:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507830656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763676.924437:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763676.924441:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763676.924446:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763676.924453:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830784:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763676.924478:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763676.924491:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763676/real 1789763676] req@ffff8d25c8022d80 x1876699507830784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763676.924505:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507830784/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763676.924515:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763676.924519:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763676.924522:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507830784:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763676.924529:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507830912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763676.924548:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763676.924560:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763676/real 1789763676] req@ffff8d25c8023100 x1876699507830912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763676.924574:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507830912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763676.924583:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763676.924587:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763676.924590:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507830912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763676.924596:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507831040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763676.924621:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763676.924642:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763676/real 1789763676] req@ffff8d25c8020e00 x1876699507831040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763676.924679:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507831040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763701 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763676.924697:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763676.924704:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763676.924710:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507831040:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763676.987625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763678.011595:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763679.035570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763680.059632:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763681.083607:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763682.043581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763682.043592:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763682.043597:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763682.043609:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3659 00000100:00080000:0.0:1789763682.043619:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763682.043628:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763682.043667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507831168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763682.043699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763682.043704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763682.043706:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763682.043711:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3659 00000100:00080000:0.0:1789763682.043715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763682.043720:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763682.043731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763682.043746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507831296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763682.043757:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507831168:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763682.043765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763682.043771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763682.043773:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763682.043779:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3659 00000100:00080000:0.0:1789763682.043783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763682.043787:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763682.043793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763682.043810:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507831424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763682.043821:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763682.043826:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763682.043829:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763682.043833:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3659 00000100:00080000:0.0:1789763682.043838:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763682.043845:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763682.043849:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763682.043858:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763682.043873:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507831552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763682.043884:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763682.043888:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763682.043890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763682.043895:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3659 00000100:00080000:0.0:1789763682.043899:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763682.043904:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763682.043910:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763682.043918:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763682] req@ffff8d25d25bb800 x1876699507831168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763682.043927:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507831680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763682.043967:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763682.043972:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763682.043975:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763682.043982:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507831168/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763682.043986:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3659 00000100:00080000:0.0:1789763682.043995:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763682.044005:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789763682.044015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763682.044016:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763682.044032:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763682.044037:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507831808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763682.044048:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507831168:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763682.044136:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507831296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763682.044173:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763682.044191:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763682] req@ffff8d25d25b9f80 x1876699507831296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763682.044208:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507831296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763682.044224:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763682.044231:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763682.044238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507831296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763682.044249:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507831424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763682.044288:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763682.044311:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763682] req@ffff8d25d25bb480 x1876699507831424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763682.044329:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507831424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763682.044339:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763682.044343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763682.044347:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507831424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763682.044355:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507831552:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763682.044388:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763682.044451:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763682] req@ffff8d25d25bbb80 x1876699507831552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763682.044471:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507831552/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763682.044481:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763682.044485:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763682.044490:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507831552:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763682.044497:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507831680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763682.044525:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763682.044538:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763682] req@ffff8d25d25b9180 x1876699507831680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763682.044557:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507831680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763682.044572:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763682.044577:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763682.044584:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507831680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763682.044596:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507831808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763682.044648:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763682.044667:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763682/real 1789763682] req@ffff8d25d25b8a80 x1876699507831808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763682.044681:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507831808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763707 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763682.044690:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763682.044693:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763682.044697:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507831808:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763682.107576:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763683.131512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763684.155628:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763685.179635:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763686.203568:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763687.163597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763687.163611:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763687.163615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763687.163631:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3664 00000100:00080000:3.0:1789763687.163645:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763687.163658:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763687.163718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507831936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763687.163766:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763687.163771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763687.163774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763687.163780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3664 00000100:00080000:3.0:1789763687.163786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763687.163792:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763687.163805:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763687.163825:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507832064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763687.163838:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507831936:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763687.163846:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763687.163853:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763687.163856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763687.163862:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3664 00000100:00080000:3.0:1789763687.163867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763687.163871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763687.163878:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763687.163896:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507832192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763687.163909:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763687.163915:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763687.163918:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763687.163922:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3664 00000100:00080000:3.0:1789763687.163926:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763687.163934:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763687.163937:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763687.163949:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763687.163964:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507832320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763687.163977:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763687.163982:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763687.163985:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763687.163991:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3664 00000100:00080000:3.0:1789763687.163996:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763687.164000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffff8d25d1b09500 x1876699507831936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763687.164006:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763687.164018:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763687.164044:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507832448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763687.164081:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507831936/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763687.164091:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763687.164113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763687.164120:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763687.164122:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763687.164136:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763687.164144:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3664 00000100:00100000:2.0:1789763687.164149:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507831936:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763687.164159:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763687.164174:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763687.164187:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763687.164196:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763687.164212:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507832576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763687.164245:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763687.164273:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffff8d25d1b08000 x1876699507832064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763687.164291:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507832064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763687.164303:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763687.164307:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763687.164312:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763687.164322:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763687.164359:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763687.164431:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffff8d25d1b0a300 x1876699507832192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763687.164460:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507832192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763687.164477:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763687.164484:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763687.164491:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763687.164505:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832320:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763687.164542:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763687.164558:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffff8d25d1b0b100 x1876699507832320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763687.164573:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507832320/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763687.164585:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763687.164589:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763687.164593:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832320:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763687.164599:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507832448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763687.164638:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763687.164652:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffff8d25d1b08380 x1876699507832448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763687.164666:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507832448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763687.164676:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763687.164680:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763687.164684:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507832448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763687.164710:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507832576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763687.164730:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763687.164742:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763687/real 1789763687] req@ffff8d25d1b09500 x1876699507832576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763687.164756:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507832576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763712 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763687.164766:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763687.164769:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763687.164774:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507832576:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763687.227603:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763688.251609:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763689.275631:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763690.299605:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763691.323570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763692.283602:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763692.283613:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763692.283618:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763692.283632:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3669 00000100:00080000:1.0:1789763692.283644:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763692.283653:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763692.283700:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507832704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763692.283740:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763692.283745:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763692.283748:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763692.283754:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3669 00000100:00080000:1.0:1789763692.283759:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763692.283765:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763692.283778:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763692.283803:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507832832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763692.283803:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507832704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763692.283831:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763692.283836:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763692.283838:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763692.283843:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3669 00000100:00080000:1.0:1789763692.283847:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763692.283852:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763692.283859:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763692.283874:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507832960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763692.283887:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763692.283891:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763692.283893:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763692.283903:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3669 00000800:00000100:2.0:1789763692.283904:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763692.283914:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763692.283922:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763692.283929:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763692.283948:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507833088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763692.283960:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763692.283964:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763692.283966:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763692.283971:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3669 00000100:00080000:1.0:1789763692.283976:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763692.283981:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763692.283990:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffff8d25c8168700 x1876699507832704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:1.0:1789763692.283992:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763692.284048:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507833216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763692.284054:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507832704/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763692.284110:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763692.284113:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763692.284124:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789763692.284125:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763692.284133:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763692.284138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507832704:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763692.284146:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3669 00000100:00080000:1.0:1789763692.284156:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763692.284161:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763692.284168:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763692.284188:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763692.284189:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507833344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763692.284247:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763692.284268:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffff8d25c8168380 x1876699507832832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763692.284285:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507832832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763692.284296:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763692.284300:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763692.284305:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763692.284312:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763692.284335:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763692.284348:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffff8d25c8168e00 x1876699507832960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763692.284362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507832960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763692.284372:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763692.284412:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763692.284419:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507832960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763692.284431:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833088:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763692.284473:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763692.284497:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffff8d25c8169f80 x1876699507833088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763692.284521:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507833088/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763692.284531:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763692.284535:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763692.284540:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833088:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763692.284547:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507833216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763692.284574:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763692.284588:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffff8d25c8168000 x1876699507833216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763692.284611:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507833216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763692.284621:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763692.284624:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763692.284628:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507833216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763692.284652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507833344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763692.284672:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763692.284685:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763692/real 1789763692] req@ffff8d25c8169500 x1876699507833344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763692.284699:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507833344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763717 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763692.284709:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763692.284712:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763692.284715:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507833344:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763692.347603:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763693.371637:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763694.395617:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763695.419518:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763696.443608:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763697.403600:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763697.403613:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763697.403618:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763697.403631:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3675 00000100:00080000:0.0:1789763697.403644:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763697.403653:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763697.403701:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507833472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763697.403744:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763697.403750:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763697.403752:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763697.403758:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3675 00000100:00080000:0.0:1789763697.403762:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763697.403767:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763697.403778:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763697.403794:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507833600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763697.403807:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763697.403812:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763697.403815:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763697.403820:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3675 00000100:00080000:0.0:1789763697.403824:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763697.403828:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763697.403839:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763697.403839:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507833472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763697.403862:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507833728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763697.403876:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763697.403881:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763697.403884:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763697.403890:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3675 00000100:00080000:0.0:1789763697.403895:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763697.403900:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763697.403907:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763697.403921:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507833856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763697.403937:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763697.403937:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763697.403948:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763697.403952:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763697.403959:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3675 00000100:00080000:0.0:1789763697.403966:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763697.404003:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763697.404010:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763697.404029:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507833984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763697.404040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763697/real 1789763697] req@ffff8d25d25bad80 x1876699507833472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763697.404052:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763697.404066:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763697.404072:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763697.404086:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3675 00000100:00080000:0.0:1789763697.404092:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763697.404098:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763697.404106:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507833472/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763697.404108:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763697.404131:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507834112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763697.404135:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763697.404155:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763697.404165:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507833472:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763697.404190:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763697.404227:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763697.404246:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763697/real 1789763697] req@ffff8d25d25ba300 x1876699507833600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763697.404262:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507833600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763697.404273:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763697.404278:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763697.404282:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763697.404312:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763697.404334:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763697.404347:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763697/real 1789763697] req@ffff8d25d25b8000 x1876699507833728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763697.404362:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507833728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763697.404372:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763697.404409:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763697.404414:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763697.404435:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833856:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763697.404461:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763697.404476:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763697/real 1789763697] req@ffff8d25d25b8a80 x1876699507833856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763697.404491:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507833856/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763697.404501:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763697.404505:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763697.404509:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507833856:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763697.404516:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507833984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763697.404535:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763697.404546:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763697/real 1789763697] req@ffff8d25d25b9180 x1876699507833984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763697.404560:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507833984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763697.404570:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763697.404573:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763697.404577:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507833984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763697.404583:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507834112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763697.404602:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763697.404620:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763697/real 1789763697] req@ffff8d25d25bbb80 x1876699507834112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763697.404644:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507834112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763722 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763697.404659:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763697.404666:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763697.404673:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507834112:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763697.467616:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763698.491619:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763699.515652:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763700.539600:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763701.563611:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763702.523485:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763702.523494:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763702.523499:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763702.523520:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3680 00000100:00080000:0.0:1789763702.523529:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763702.523537:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763702.523576:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507834240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763702.523604:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763702.523608:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763702.523610:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763702.523614:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3680 00000100:00080000:0.0:1789763702.523620:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763702.523626:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763702.523636:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763702.523652:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507834368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763702.523662:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763702.523666:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763702.523668:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763702.523678:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3680 00000100:00100000:2.0:1789763702.523679:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507834240:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763702.523681:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763702.523694:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763702.523700:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763702.523713:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507834496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763702.523722:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763702.523725:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763702.523727:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763702.523731:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3680 00000100:00080000:0.0:1789763702.523734:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763702.523737:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763702.523741:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763702.523751:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507834624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763702.523756:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763702.523768:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763702.523771:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763702.523774:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763702.523780:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3680 00000100:00080000:0.0:1789763702.523782:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763702.523786:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763702.523790:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763702.523811:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763702/real 1789763702] req@ffff8d25d25bb480 x1876699507834240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763702.523812:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507834752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763702.523854:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763702.523858:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763702.523861:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763702.523864:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507834240/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763702.523869:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3680 00000100:00080000:0.0:1789763702.523877:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763702.523885:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763702.523888:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763702.523890:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763702.523898:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763702.523906:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507834240:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763702.523909:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507834880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763702.523928:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507834368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763702.523981:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763702.523999:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763702/real 1789763702] req@ffff8d25d25b9f80 x1876699507834368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763702.524017:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507834368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763702.524028:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763702.524033:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763702.524039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507834368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763702.524073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507834496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763702.524087:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763702.524096:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763702/real 1789763702] req@ffff8d25d25bb800 x1876699507834496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763702.524105:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507834496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763702.524112:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763702.524114:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763702.524116:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507834496:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763702.524121:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507834624:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763702.524133:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763702.524150:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763702/real 1789763702] req@ffff8d25d25b9c00 x1876699507834624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763702.524160:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507834624/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763702.524166:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763702.524168:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763702.524172:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507834624:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763702.524176:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507834752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763702.524190:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763702.524198:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763702/real 1789763702] req@ffff8d25d25ba680 x1876699507834752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763702.524209:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507834752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763702.524216:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763702.524219:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763702.524224:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507834752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763702.524229:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507834880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763702.524246:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763702.524255:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763702/real 1789763702] req@ffff8d25d25bb100 x1876699507834880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763702.524264:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507834880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763727 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763702.524270:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763702.524273:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763702.524275:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507834880:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763702.587624:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763703.611613:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763704.635546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763705.659500:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763706.683755:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763707.643546:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763707.643564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763707.643573:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763707.643596:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3685 00000100:00080000:3.0:1789763707.643617:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763707.643633:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763707.643722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507835008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763707.643765:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763707.643771:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763707.643774:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763707.643780:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3685 00000100:00080000:3.0:1789763707.643786:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763707.643794:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763707.643824:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763707.643846:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507835008:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763707.643849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507835136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763707.643891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763707.643938:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763707.643942:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763707.643951:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3685 00000100:00080000:3.0:1789763707.643958:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763707.643965:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763707.643973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763707.643992:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507835264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763707.644005:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763707.644009:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763707.644012:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763707.644022:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3685 00000800:00000100:2.0:1789763707.644032:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763707.644034:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763707.644049:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763707.644057:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763707.644072:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507835392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763707.644086:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763707.644091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763707.644095:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763707.644101:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3685 00000100:00080000:3.0:1789763707.644108:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763707.644115:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763707.644122:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763707.644125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763707/real 1789763707] req@ffff8d25d1b0b480 x1876699507835008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763707.644144:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507835520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763707.644169:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763707.644175:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507835008/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763707.644177:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763707.644189:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763707.644205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763707.644216:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3685 00000100:00080000:2.0:1789763707.644227:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763707.644229:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763707.644251:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507835008:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763707.644253:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763707.644277:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763707.644301:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507835648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763707.644312:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763707.644425:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763707.644447:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763707/real 1789763707] req@ffff8d25d1b08a80 x1876699507835136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763707.644464:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507835136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763707.644475:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763707.644480:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763707.644485:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763707.644493:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763707.644517:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763707.644530:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763707/real 1789763707] req@ffff8d25d1b0ad80 x1876699507835264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763707.644553:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507835264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763707.644563:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763707.644567:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763707.644570:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763707.644578:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835392:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763707.644598:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763707.644610:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763707/real 1789763707] req@ffff8d25d1b09180 x1876699507835392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763707.644625:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507835392/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763707.644634:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763707.644638:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763707.644641:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835392:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763707.644648:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507835520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763707.644677:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763707.644688:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763707/real 1789763707] req@ffff8d25d1b0aa00 x1876699507835520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763707.644703:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507835520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763707.644720:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763707.644724:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763707.644728:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507835520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763707.644757:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507835648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763707.644778:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763707.644791:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763707/real 1789763707] req@ffff8d25d1b09c00 x1876699507835648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763707.644805:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507835648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763732 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763707.644815:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763707.644819:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763707.644822:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507835648:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763707.707612:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763708.731620:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763709.755537:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763710.779638:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763711.803658:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763712.763455:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763712.763465:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763712.763470:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763712.763481:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3690 00000100:00080000:3.0:1789763712.763491:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763712.763498:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763712.763550:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507835776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763712.763575:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763712.763578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763712.763579:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763712.763583:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3690 00000100:00080000:3.0:1789763712.763585:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763712.763589:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763712.763600:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763712.763609:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507835904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763712.763615:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507835776:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763712.763619:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763712.763625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763712.763627:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763712.763631:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3690 00000100:00080000:3.0:1789763712.763633:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763712.763635:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763712.763638:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763712.763646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507836032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763712.763652:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763712.763654:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763712.763655:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763712.763659:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3690 00000100:00080000:3.0:1789763712.763662:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763712.763664:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763712.763668:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763712.763675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507836160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763712.763681:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763712.763683:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763712.763685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763712.763692:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3690 00000100:00080000:3.0:1789763712.763695:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763712.763698:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763712.763702:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763712.763705:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763712.763712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507836288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763712.763720:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763712.763723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763712.763725:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763712.763727:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3690 00000100:00080000:3.0:1789763712.763730:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763712.763732:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763712.763735:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763712.763743:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507836416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763712.763755:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763712/real 1789763712] req@ffff8d25d1b0bb80 x1876699507835776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763712.763771:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507835776/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763712.763780:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763712.763784:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763712.763787:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507835776:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763712.763809:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763712.763827:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763712.763836:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763712/real 1789763712] req@ffff8d25d1b0a680 x1876699507835904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763712.763845:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507835904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763712.763854:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763712.763870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763712.763875:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507835904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763712.763882:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763712.763925:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763712.763949:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763712/real 1789763712] req@ffff8d25d1b08700 x1876699507836032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763712.763966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507836032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763712.763977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763712.763981:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763712.763985:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763712.763993:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836160:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763712.764020:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763712.764034:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763712/real 1789763712] req@ffff8d25d1b08e00 x1876699507836160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763712.764049:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507836160/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763712.764060:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763712.764077:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763712.764082:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836160:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763712.764089:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507836288:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763712.764112:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763712.764125:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763712/real 1789763712] req@ffff8d25d1b09880 x1876699507836288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763712.764140:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507836288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763712.764156:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763712.764162:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763712.764169:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507836288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763712.764182:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507836416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763712.764235:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763712.764248:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763712/real 1789763712] req@ffff8d25d1b0b800 x1876699507836416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763712.764266:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507836416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763737 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763712.764274:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763712.764278:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763712.764281:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507836416:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763712.827541:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763713.851579:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763714.875632:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763715.899637:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763716.923612:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763717.883627:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763717.883645:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763717.883651:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763717.883668:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3695 00000100:00080000:1.0:1789763717.883686:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763717.883698:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763717.883764:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507836544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763717.883809:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763717.883815:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763717.883817:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763717.883850:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3695 00000100:00080000:1.0:1789763717.883858:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763717.883865:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763717.883882:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763717.883904:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507836544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763717.883906:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507836672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763717.883942:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763717.883948:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763717.883953:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763717.883962:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3695 00000100:00080000:1.0:1789763717.883967:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763717.883974:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763717.883981:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763717.884001:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507836800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763717.884015:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763717.884020:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763717.884033:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763717.884038:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763717.884048:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3695 00000100:00080000:1.0:1789763717.884058:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763717.884065:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763717.884073:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763717.884089:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507836928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763717.884106:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763717/real 1789763717] req@ffff8d25c816aa00 x1876699507836544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763717.884166:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763717.884178:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763717.884182:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763717.884196:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3695 00000100:00100000:2.0:1789763717.884197:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507836544/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763717.884207:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763717.884227:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763717.884242:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763717.884242:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763717.884259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763717.884273:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507836544:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763717.884275:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507837056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763717.884306:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763717.884312:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763717.884315:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763717.884322:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3695 00000100:00080000:1.0:1789763717.884327:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763717.884336:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836672:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789763717.884346:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763717.884361:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763717.884433:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507837184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763717.884452:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763717.884483:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763717/real 1789763717] req@ffff8d25c816a680 x1876699507836672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763717.884505:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507836672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763717.884518:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763717.884523:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763717.884527:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763717.884536:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763717.884558:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763717.884571:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763717/real 1789763717] req@ffff8d25c8169500 x1876699507836800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763717.884585:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507836800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763717.884595:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763717.884598:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763717.884602:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763717.884610:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836928:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763717.884629:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763717.884640:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763717/real 1789763717] req@ffff8d25c8168000 x1876699507836928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763717.884655:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507836928/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763717.884664:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763717.884668:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763717.884672:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507836928:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763717.884680:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763717.884699:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763717.884710:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763717/real 1789763717] req@ffff8d25c8169f80 x1876699507837056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763717.884724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507837056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763717.884734:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763717.884737:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763717.884741:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763717.884763:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763717.884782:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763717.884794:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763717/real 1789763717] req@ffff8d25c8168e00 x1876699507837184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763717.884807:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507837184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763742 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763717.884816:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763717.884848:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763717.884854:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837184:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763717.947651:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763718.971626:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763719.995646:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763721.019577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763722.043649:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763723.003560:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763723.003591:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763723.003598:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763723.003612:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3700 00000100:00080000:0.0:1789763723.003628:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763723.003638:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763723.003688:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507837312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763723.003726:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763723.003732:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763723.003735:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763723.003740:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3700 00000100:00080000:0.0:1789763723.003744:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763723.003749:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763723.003762:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763723.003778:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507837440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763723.003824:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763723.003830:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507837312:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763723.003831:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763723.003836:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763723.003852:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3700 00000100:00080000:0.0:1789763723.003857:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763723.003862:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763723.003869:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763723.003890:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507837568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763723.003902:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763723.003908:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763723.003910:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763723.003914:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3700 00000100:00080000:0.0:1789763723.003919:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763723.003923:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763723.003931:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:2.0:1789763723.003951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763723.003952:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507837696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763723.003972:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763723.003977:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763723.003979:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763723.003985:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3700 00000100:00080000:0.0:1789763723.003989:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763723.003994:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763723.004001:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763723.004017:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffff8d25d25b8e00 x1876699507837312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763723.004020:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507837824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763723.004068:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763723.004075:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763723.004077:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507837312/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763723.004079:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763723.004095:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3700 00000100:00080000:0.0:1789763723.004108:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763723.004114:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763723.004116:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763723.004125:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789763723.004128:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763723.004136:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507837312:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763723.004150:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507837952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763723.004178:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507837440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763723.004213:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763723.004230:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffff8d25d25b8380 x1876699507837440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763723.004264:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507837440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763723.004275:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763723.004279:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763723.004283:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507837440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763723.004290:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507837568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763723.004310:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763723.004321:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffff8d25d25b8700 x1876699507837568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763723.004343:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507837568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763723.004352:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763723.004355:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763723.004359:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507837568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763723.004365:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507837696:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763723.004382:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763723.004428:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffff8d25d25bb100 x1876699507837696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763723.004446:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507837696/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763723.004457:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763723.004461:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763723.004465:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507837696:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763723.004472:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763723.004494:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763723.004508:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffff8d25d25ba680 x1876699507837824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763723.004525:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507837824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763723.004542:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763723.004546:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763723.004550:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763723.004556:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763723.004573:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763723.004583:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763723/real 1789763723] req@ffff8d25d25b9c00 x1876699507837952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763723.004623:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507837952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763748 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763723.004633:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763723.004636:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763723.004640:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507837952:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763723.067632:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763724.091606:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763725.115586:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763726.139605:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763727.163670:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763728.123548:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.123562:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763728.123568:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.123584:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3705 00000100:00080000:3.0:1789763728.123604:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.123619:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763728.123681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507838080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763728.123724:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.123728:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763728.123731:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.123737:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3705 00000100:00080000:3.0:1789763728.123743:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.123779:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763728.123794:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763728.123815:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507838208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763728.123842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.123851:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763728.123852:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507838080:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763728.123856:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.123878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3705 00000100:00080000:3.0:1789763728.123885:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.123890:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763728.123899:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763728.123930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507838336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763728.123943:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.123948:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763728.123951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.123957:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3705 00000100:00080000:3.0:1789763728.123978:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.123985:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763728.123992:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763728.124008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507838464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763728.124023:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.124028:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763728.124031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.124036:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3705 00000100:00080000:3.0:1789763728.124041:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.124048:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763728.124057:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763728.124077:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507838592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763728.124084:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838208:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763728.124130:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763728.124136:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763728.124140:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763728.124149:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3705 00000100:00080000:3.0:1789763728.124163:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763728.124169:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763728.124182:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763728.124185:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763728.124208:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507838720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763728.124244:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763728.124288:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507838592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763728.124329:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507838720:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763728.187642:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763729.211558:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763730.235620:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763731.196123:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789763731.196213:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763731.196239:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763731.196319:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763731.196341:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763731.196357:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763731.196370:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763731.196444:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:2.0:1789763731.196598:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763731] req@ffff8d25d1b0a680 x1876699507838592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763731.196642:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507838592/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763731.196665:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763731.196674:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763731.196683:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507838592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763731.196699:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763731] req@ffff8d25d1b0bb80 x1876699507838720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763731.196753:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507838720/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763731.196771:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763731.196778:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763731.196787:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507838720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763731.196801:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763731] req@ffff8d25d1b09880 x1876699507838208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763731.196826:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507838208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763731.196843:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763731.196852:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763731.196859:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763731.196873:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763731] req@ffff8d25d1b08e00 x1876699507838336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763731.196898:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507838336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763731.196914:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763731.196921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763731.196928:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763731.196941:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763731] req@ffff8d25d1b08700 x1876699507838464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763731.196966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507838464/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763731.196984:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763731.196991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763731.196998:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838464:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763731.197012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763728/real 1789763731] req@ffff8d25d1b0b800 x1876699507838080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763731.197036:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507838080/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763753 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763731.197052:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763731.197059:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763731.197065:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507838080:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789763731.259543:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763732.283571:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763733.243555:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763733.243571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763733.243578:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.243594:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3710 00000100:00080000:1.0:1789763733.243611:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.243625:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763733.243683:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507838848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763733.243748:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763733.243757:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763733.243760:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.243769:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3710 00000100:00080000:1.0:1789763733.243777:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.243784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763733.243799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763733.243818:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507838848:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763733.243824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507838976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763733.243842:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763733.243848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763733.243850:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.243857:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3710 00000100:00080000:1.0:1789763733.243862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.243867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763733.243874:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763733.243890:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507839104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763733.243902:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763733.243906:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763733.243912:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763733.243915:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.243921:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3710 00000100:00080000:1.0:1789763733.243929:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.243946:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763733.243954:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763733.243958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffff8d25c8168380 x1876699507838848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763733.243972:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507839232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763733.243987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507838848/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763733.243989:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763733.243996:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763733.243999:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.244005:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3710 00000100:00080000:3.0:1789763733.244006:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763733.244011:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763733.244013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:1.0:1789763733.244017:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763733.244020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507838848:192.168.203.141@tcp:250:kworker.0 00000008:00100000:1.0:1789763733.244025:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763733.244040:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507839360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763733.244055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763733.244055:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838976:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789763733.244062:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763733.244066:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763733.244072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3710 00000100:00080000:1.0:1789763733.244077:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763733.244083:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763733.244092:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763733.244098:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763733.244123:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507839488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763733.244123:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffff8d25c8168700 x1876699507838976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763733.244148:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507838976/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763733.244163:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763733.244170:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763733.244177:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507838976:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763733.244189:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839104:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763733.244228:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763733.244250:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffff8d25c8168a80 x1876699507839104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763733.244272:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507839104/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763733.244287:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763733.244292:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763733.244299:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763733.244310:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839232:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763733.244348:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763733.244370:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffff8d25c816a300 x1876699507839232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763733.244447:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507839232/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763733.244461:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763733.244467:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763733.244473:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839232:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763733.244507:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507839360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763733.244548:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763733.244571:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffff8d25c816ad80 x1876699507839360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763733.244591:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507839360/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763733.244605:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763733.244610:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763733.244615:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507839360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763733.244625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507839488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763733.244839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763733.244882:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763733/real 1789763733] req@ffff8d25c8169c00 x1876699507839488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763733.244908:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507839488/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763758 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763733.244936:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763733.244943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763733.244958:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507839488:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763733.307640:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763734.331612:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763735.355616:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763736.379731:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763737.403553:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763738.363540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763738.363553:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763738.363560:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.363574:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3716 00000100:00080000:3.0:1789763738.363591:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.363602:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763738.363650:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507839616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763738.363725:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763738.363731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763738.363734:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.363742:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3716 00000100:00080000:3.0:1789763738.363748:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.363755:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763738.363767:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763738.363799:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507839744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763738.363812:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507839616:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763738.363820:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763738.363827:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763738.363830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.363839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3716 00000100:00080000:3.0:1789763738.363856:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.363862:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763738.363871:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763738.363892:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507839872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763738.363923:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763738.363932:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000800:00000100:2.0:1789763738.363933:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763738.363939:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.363948:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3716 00000100:00080000:3.0:1789763738.363955:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.363961:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763738.363967:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763738.363984:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507840000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763738.363997:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763738.364000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffff8d25d1b09c00 x1876699507839616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763738.364004:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763738.364027:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507839616/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763738.364041:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763738.364047:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763738.364052:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507839616:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763738.364085:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839744:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763738.364103:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763738.364112:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3716 00000100:00080000:3.0:1789763738.364124:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763738.364132:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763738.364137:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763738.364148:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763738.364160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffff8d25d1b0aa00 x1876699507839744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763738.364178:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507839744/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763738.364193:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763738.364197:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763738.364201:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839744:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763738.364210:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763738.364232:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763738.364281:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffff8d25d1b09180 x1876699507839872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763738.364301:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507839872/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763738.364310:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763738.364314:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763738.364319:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507839872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763738.364326:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840000:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763738.364356:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763738.364451:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffff8d25d1b0ad80 x1876699507840000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763738.364492:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507840000/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763738.364510:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763738.364517:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763738.364525:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840000:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763738.364545:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507840128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763738.364592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763738.364601:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763738.364606:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763738.364615:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507840128:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763738.364619:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3716 00000100:00080000:3.0:1789763738.364636:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763738.364647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763738.364660:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763738.364663:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763738.364713:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffff8d25d1b08a80 x1876699507840128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763738.364746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507840256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763738.364758:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507840128/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763738.364792:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763738.364798:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763738.364804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507840128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763738.364825:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507840256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763738.364852:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763738.364864:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763738/real 1789763738] req@ffff8d25d1b0b480 x1876699507840256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763738.364877:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507840256/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763763 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763738.364886:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763738.364889:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763738.364892:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507840256:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763738.427554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763739.451577:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763740.475593:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763741.499583:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763742.523680:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763743.483584:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763743.483597:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763743.483602:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763743.483615:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3721 00000100:00080000:0.0:1789763743.483627:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763743.483663:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763743.483713:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507840384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763743.483764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763743.483773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763743.483777:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763743.483786:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3721 00000100:00080000:0.0:1789763743.483795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763743.483806:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763743.483822:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763743.483837:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507840384:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763743.483848:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507840512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763743.483867:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763743.483873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763743.483875:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763743.483881:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3721 00000100:00080000:0.0:1789763743.483887:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763743.483892:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763743.483900:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763743.483916:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507840640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763743.483930:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763743.483932:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763743.483942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763743.483946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763743.483954:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3721 00000100:00080000:0.0:1789763743.483960:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763743.483965:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763743.483973:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763743.483988:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507840768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763743.484003:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763743/real 1789763743] req@ffff8d25d25bb800 x1876699507840384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763743.484007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763743.484019:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763743.484023:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763743.484039:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3721 00000100:00080000:0.0:1789763743.484048:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763743.484059:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763743.484069:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763743.484091:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507840896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763743.484107:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763743.484109:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507840384/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763743.484111:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763743.484118:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763743.484134:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3721 00000100:00080000:0.0:1789763743.484150:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763743.484153:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763743.484162:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763743.484168:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763743.484181:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507840384:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789763743.484182:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763743.484219:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507841024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763743.484225:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763743.484271:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763743.484295:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763743/real 1789763743] req@ffff8d25d25b9f80 x1876699507840512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763743.484313:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507840512/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763743.484324:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763743.484329:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763743.484333:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840512:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763743.484340:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840640:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763743.484363:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763743.484430:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763743/real 1789763743] req@ffff8d25d25bb480 x1876699507840640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763743.484458:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507840640/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763743.484484:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763743.484488:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763743.484493:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763743.484502:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840768:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763743.484533:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763743.484550:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763743/real 1789763743] req@ffff8d25d25bbb80 x1876699507840768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763743.484564:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507840768/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763743.484575:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763743.484579:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763743.484583:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507840768:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763743.484589:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507840896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763743.484609:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763743.484621:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763743/real 1789763743] req@ffff8d25d25b9180 x1876699507840896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763743.484661:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507840896/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763743.484672:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763743.484675:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763743.484681:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507840896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763743.484706:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507841024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763743.484734:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763743.484748:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763743/real 1789763743] req@ffff8d25d25b8a80 x1876699507841024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763743.484763:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507841024/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763768 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763743.484773:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763743.484777:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763743.484781:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507841024:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763743.547630:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763744.571683:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763745.595655:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763746.619581:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763747.643671:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763748.603537:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763748.603551:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763748.603557:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763748.603572:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3726 00000100:00080000:0.0:1789763748.603616:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763748.603628:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763748.603675:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507841152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763748.603714:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763748.603734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763748.603738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763748.603746:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3726 00000100:00080000:0.0:1789763748.603754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763748.603762:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763748.603779:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763748.603796:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507841152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763748.603803:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507841280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763748.603823:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763748.603829:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763748.603832:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763748.603839:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3726 00000100:00080000:0.0:1789763748.603845:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763748.603851:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763748.603860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763748.603877:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507841408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763748.603883:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763748.603896:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763748.603901:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763748.603904:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763748.603911:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3726 00000100:00080000:0.0:1789763748.603917:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763748.603923:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763748.603931:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763748.603944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763748/real 1789763748] req@ffff8d25d25b8000 x1876699507841152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763748.603951:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507841536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763748.603970:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763748.603976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763748.603980:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763748.603986:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3726 00000100:00080000:0.0:1789763748.603992:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763748.603996:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507841152/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763748.603999:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:2.0:1789763748.604016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763748.604019:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763748.604024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763748.604032:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507841152:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763748.604038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507841664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763748.604055:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763748.604061:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763748.604064:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763748.604072:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3726 00000100:00100000:2.0:1789763748.604073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507841280:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763748.604079:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763748.604085:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763748.604094:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763748.604116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763748.604127:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507841792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763748.604142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763748/real 1789763748] req@ffff8d25d25ba300 x1876699507841280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763748.604165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507841280/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763748.604183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763748.604190:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763748.604197:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507841280:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763748.604210:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507841408:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763748.604248:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763748.604266:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763748/real 1789763748] req@ffff8d25d25bad80 x1876699507841408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763748.604281:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507841408/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763748.604292:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763748.604296:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763748.604300:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507841408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763748.604307:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507841536:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763748.604328:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763748.604340:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763748/real 1789763748] req@ffff8d25d25b8a80 x1876699507841536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763748.604355:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507841536/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763748.604365:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763748.604369:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763748.604372:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507841536:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763748.604439:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507841664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763748.604481:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763748.604506:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763748/real 1789763748] req@ffff8d25d25b9180 x1876699507841664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763748.604531:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507841664/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763748.604548:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763748.604555:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763748.604560:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507841664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763748.604587:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507841792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763748.604656:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763748.604673:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763748/real 1789763748] req@ffff8d25d25bbb80 x1876699507841792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763748.604688:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507841792/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763773 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763748.604707:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763748.604712:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763748.604716:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507841792:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763748.667543:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763749.691637:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763750.715641:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763751.739605:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763752.763725:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763753.723574:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763753.723585:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763753.723589:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763753.723600:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3731 00000100:00080000:1.0:1789763753.723611:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763753.723619:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763753.723658:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507841920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763753.723684:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763753.723688:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763753.723690:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763753.723694:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3731 00000100:00080000:1.0:1789763753.723696:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763753.723700:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763753.723708:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763753.723718:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507842048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763753.723728:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763753.723731:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763753.723736:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763753.723739:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3731 00000100:00080000:1.0:1789763753.723742:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763753.723745:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763753.723750:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763753.723761:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507842176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763753.723771:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763753.723774:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763753.723776:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763753.723779:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3731 00000100:00080000:1.0:1789763753.723782:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763753.723787:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763753.723794:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763753.723802:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507841920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763753.723808:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507842304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763753.723836:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763753.723839:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763753.723841:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763753.723844:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3731 00000100:00080000:1.0:1789763753.723848:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763753.723852:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763753.723858:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763753.723873:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507842432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763753.723887:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763753.723890:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763753.723893:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763753.723896:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3731 00000100:00080000:1.0:1789763753.723901:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763753.723904:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763753.723907:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763753.723917:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507842560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763753.723937:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763753.724022:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763753/real 1789763753] req@ffff8d25c8169880 x1876699507841920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763753.724047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507841920/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763753.724061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763753.724066:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763753.724071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507841920:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763753.724081:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763753.724132:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763753.724155:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763753/real 1789763753] req@ffff8d25c816b480 x1876699507842048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763753.724192:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507842048/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763753.724220:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763753.724227:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763753.724233:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842048:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763753.724268:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842176:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763753.724297:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763753.724312:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763753/real 1789763753] req@ffff8d25c8169c00 x1876699507842176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763753.724325:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507842176/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763753.724334:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763753.724337:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763753.724341:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763753.724347:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842304:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763753.724365:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763753.724419:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763753/real 1789763753] req@ffff8d25c816ad80 x1876699507842304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763753.724434:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507842304/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763753.724443:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763753.724447:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763753.724451:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842304:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763753.724459:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507842432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763753.724483:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763753.724521:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763753/real 1789763753] req@ffff8d25c816a300 x1876699507842432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763753.724545:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507842432/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763753.724588:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763753.724613:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763753.724620:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507842432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763753.724631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507842560:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763753.724680:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763753.724702:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763753/real 1789763753] req@ffff8d25c8168a80 x1876699507842560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763753.724727:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507842560/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763778 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763753.724743:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763753.724750:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763753.724757:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507842560:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763753.787544:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763754.811547:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763755.835689:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763756.859668:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763757.883554:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763758.843600:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763758.843613:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763758.843618:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763758.843631:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3736 00000100:00080000:1.0:1789763758.843643:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763758.843652:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763758.843711:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507842688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763758.843747:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763758.843752:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763758.843755:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763758.843760:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3736 00000100:00080000:1.0:1789763758.843764:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763758.843768:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763758.843782:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763758.843798:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507842816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763758.843813:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763758.843823:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763758.843823:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507842688:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763758.843831:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763758.843847:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3736 00000100:00080000:1.0:1789763758.843850:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763758.843854:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763758.843862:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763758.843878:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507842944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763758.843891:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763758.843896:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763758.843898:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763758.843902:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3736 00000100:00080000:1.0:1789763758.843926:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763758.843927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763758.843944:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763758.843955:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763758.843982:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507843072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763758.844011:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763758/real 1789763758] req@ffff8d25c8168700 x1876699507842688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763758.844019:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763758.844039:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763758.844047:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763758.844061:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3736 00000100:00080000:1.0:1789763758.844068:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763758.844075:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507842688/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763758.844104:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763758.844119:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763758.844119:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763758.844137:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763758.844151:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507842688:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763758.844152:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507843200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763758.844173:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763758.844178:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763758.844182:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763758.844188:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3736 00000100:00080000:1.0:1789763758.844194:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763758.844198:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842816:192.168.203.141@tcp:38:kworker.0 00000100:00080000:1.0:1789763758.844206:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763758.844216:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763758.844234:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507843328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763758.844254:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763758.844279:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763758/real 1789763758] req@ffff8d25c8168380 x1876699507842816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763758.844302:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507842816/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763758.844314:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763758.844318:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763758.844322:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842816:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763758.844329:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763758.844349:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763758.844363:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763758/real 1789763758] req@ffff8d25c8168e00 x1876699507842944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763758.844428:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507842944/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763758.844438:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763758.844442:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763758.844447:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507842944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763758.844454:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843072:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763758.844478:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763758.844492:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763758/real 1789763758] req@ffff8d25c8169f80 x1876699507843072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763758.844507:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507843072/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763758.844557:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763758.844565:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763758.844572:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843072:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763758.844586:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507843200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763758.844626:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763758.844650:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763758/real 1789763758] req@ffff8d25c8168000 x1876699507843200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763758.844673:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507843200/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763758.844690:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763758.844694:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763758.844698:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507843200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763758.844726:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507843328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763758.844754:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763758.844769:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763758/real 1789763758] req@ffff8d25c8169500 x1876699507843328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763758.844783:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507843328/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763783 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763758.844793:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763758.844797:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763758.844801:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507843328:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763758.907641:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763759.931686:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763760.955540:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763761.979629:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763763.003676:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763763.963569:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763763.963578:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763763.963582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763763.963590:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3741 00000100:00080000:0.0:1789763763.963598:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763763.963606:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763763.963646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507843456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763763.963677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763763.963679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763763.963681:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763763.963684:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3741 00000100:00080000:0.0:1789763763.963687:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763763.963691:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763763.963698:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763763.963709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507843584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763763.963718:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763763.963723:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:2.0:1789763763.963723:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507843456:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763763.963728:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763763.963735:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3741 00000100:00080000:0.0:1789763763.963737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763763.963739:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763763.963742:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763763.963751:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507843712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763763.963758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763763.963760:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763763.963761:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763763.963764:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3741 00000100:00080000:0.0:1789763763.963766:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763763.963768:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763763.963772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763763.963780:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507843840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763763.963787:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763763.963793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763763.963797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763763.963798:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763763.963802:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3741 00000100:00080000:0.0:1789763763.963811:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763763.963814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763763.963818:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763763.963827:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507843968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763763.963832:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763763/real 1789763763] req@ffff8d25d25bb480 x1876699507843456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763763.963837:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763763.963843:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763763.963846:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763763.963853:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3741 00000100:00080000:0.0:1789763763.963858:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763763.963861:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763763.963864:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763763.963870:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507843456/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763763.963872:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507844096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763763.963912:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763763.963918:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763763.963922:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507843456:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763763.963946:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763763.963971:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763763.963985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763763/real 1789763763] req@ffff8d25d25b9f80 x1876699507843584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763763.964011:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507843584/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763763.964020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763763.964023:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763763.964027:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843584:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763763.964033:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843712:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763763.964053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763763.964065:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763763/real 1789763763] req@ffff8d25d25bb800 x1876699507843712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763763.964077:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507843712/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763763.964086:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763763.964089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763763.964092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763763.964099:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843840:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763763.964116:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763763.964126:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763763/real 1789763763] req@ffff8d25d25b9c00 x1876699507843840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763763.964137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507843840/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763763.964145:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763763.964149:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763763.964152:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507843840:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763763.964158:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507843968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763763.964174:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763763.964185:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763763/real 1789763763] req@ffff8d25d25ba680 x1876699507843968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763763.964196:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507843968/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763763.964205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763763.964208:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763763.964212:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507843968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763763.964218:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507844096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763763.964236:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763763.964244:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763763/real 1789763763] req@ffff8d25d25bb100 x1876699507844096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763763.964252:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507844096/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763788 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763763.964257:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763763.964259:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763763.964261:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507844096:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763764.027606:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763765.051606:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763766.075627:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763767.099643:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763768.123502:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763769.083587:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763769.083599:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763769.083604:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763769.083633:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3746 00000100:00080000:2.0:1789763769.083646:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763769.083656:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763769.083703:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507844224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763769.083742:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763769.083747:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763769.083750:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763769.083755:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3746 00000100:00080000:2.0:1789763769.083760:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763769.083766:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763769.083778:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763769.083796:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507844352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763769.083809:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507844224:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763769.083816:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763769.083823:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763769.083826:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763769.083833:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3746 00000100:00080000:2.0:1789763769.083837:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763769.083842:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763769.083848:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763769.083863:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507844480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763769.083875:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763769.083880:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763769.083882:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763769.083886:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3746 00000100:00080000:2.0:1789763769.083907:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763769.083915:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763769.083925:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763769.083936:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763769.083973:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507844608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763769.083992:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763769.083999:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763769.084002:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763769.084011:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3746 00000100:00080000:2.0:1789763769.084018:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763769.084026:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffff8d25c8020700 x1876699507844224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763769.084028:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763769.084039:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763769.084079:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507844736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763769.084095:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763769.084098:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507844224/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763769.084101:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763769.084107:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763769.084120:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3746 00000100:00080000:2.0:1789763769.084131:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763769.084142:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763769.084150:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763769.084150:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.084170:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763769.084180:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507844864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763769.084182:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507844224:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763769.084230:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507844352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763769.084306:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763769.084345:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffff8d25c8021880 x1876699507844352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.084372:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507844352/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.084390:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.084397:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763769.084404:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507844352:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763769.084417:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507844480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763769.084541:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763769.084560:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffff8d25c8021c00 x1876699507844480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.084599:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507844480/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.084615:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.084622:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763769.084629:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507844480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763769.084642:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507844608:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763769.084682:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763769.084699:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffff8d25c8023480 x1876699507844608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.084724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507844608/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.084734:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.084738:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763769.084742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507844608:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763769.084750:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507844736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763769.084773:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763769.084785:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffff8d25c8021500 x1876699507844736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.084799:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507844736/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.084809:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.084812:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763769.084816:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507844736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763769.084822:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507844864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763769.084843:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763769.084862:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763769/real 1789763769] req@ffff8d25c8020e00 x1876699507844864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763769.084885:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507844864/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763794 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763769.084914:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763769.084937:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763769.084944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507844864:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763769.147648:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763770.171584:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763771.195599:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763772.219598:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763773.243652:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763774.203574:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763774.203594:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763774.203601:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763774.203615:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3751 00000100:00080000:2.0:1789763774.203630:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763774.203640:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763774.203688:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507844992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763774.203733:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763774.203738:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763774.203741:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763774.203748:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3751 00000100:00080000:2.0:1789763774.203754:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763774.203767:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763774.203779:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763774.203805:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507845120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763774.203831:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763774.203837:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507844992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763774.203839:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763774.203847:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763774.203857:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3751 00000100:00080000:2.0:1789763774.203864:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763774.203870:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763774.203879:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763774.203901:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507845248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763774.203916:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763774.203921:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763774.203924:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:1.0:1789763774.203935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763774.203936:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3751 00000100:00080000:2.0:1789763774.203947:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763774.203955:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763774.203963:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763774.203983:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507845376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763774.204004:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffff8d25c8023100 x1876699507844992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763774.204024:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763774.204035:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507844992/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763774.204038:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763774.204048:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763774.204059:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763774.204067:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3751 00000100:00080000:1.0:1789763774.204073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763774.204082:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763774.204097:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507844992:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763774.204104:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763774.204120:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763774.204136:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763774.204166:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507845504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:1.0:1789763774.204179:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763774.204203:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763774.204209:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763774.204213:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763774.204221:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3751 00000100:00080000:2.0:1789763774.204227:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763774.204229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffff8d25c8022d80 x1876699507845120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763774.204241:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763774.204261:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763774.204272:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507845120/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763774.204299:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:2.0:1789763774.204304:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507845632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763774.204311:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763774.204327:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845120:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763774.204345:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763774.204451:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763774.204496:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffff8d25c8020380 x1876699507845248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763774.204529:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507845248/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763774.204543:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763774.204549:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763774.204554:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763774.204562:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845376:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763774.204593:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763774.204608:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffff8d25c8022300 x1876699507845376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763774.204623:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507845376/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763774.204632:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763774.204636:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763774.204640:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845376:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763774.204661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507845504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763774.204681:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763774.204693:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffff8d25c8023b80 x1876699507845504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763774.204707:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507845504/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763774.204717:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763774.204721:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763774.204724:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507845504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763774.204731:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507845632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763774.204749:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763774.204761:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763774/real 1789763774] req@ffff8d25c8022680 x1876699507845632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763774.204791:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507845632/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763799 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763774.204800:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763774.204803:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763774.204807:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507845632:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763774.267619:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763775.291735:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763776.315622:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763777.339557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763778.363599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763779.323477:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.323488:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763779.323493:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.323504:0:12951:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3757 00000100:00080000:2.0:1789763779.323515:0:12951:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.323522:0:12951:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763779.323568:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507845760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763779.323593:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.323596:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763779.323597:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.323601:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3757 00000100:00080000:2.0:1789763779.323605:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.323608:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763779.323619:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763779.323624:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507845760:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763779.323634:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507845888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763779.323643:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.323646:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763779.323647:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.323651:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3757 00000100:00080000:2.0:1789763779.323654:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.323657:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763779.323661:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763779.323668:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507846016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763779.323675:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.323677:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763779.323679:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.323681:0:12951:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3757 00000100:00080000:2.0:1789763779.323684:0:12951:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.323687:0:12951:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763779.323690:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763779.323698:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763779.323700:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507846144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763779.323709:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.323712:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763779.323714:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.323717:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3757 00000100:00080000:2.0:1789763779.323719:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.323722:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763779.323725:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763779.323732:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507846272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763779.323739:0:12951:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763779.323742:0:12951:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763779.323744:0:12951:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763779.323749:0:12951:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3757 00000100:00100000:1.0:1789763779.323751:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffff8d25c8020000 x1876699507845760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763779.323751:0:12951:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763779.323756:0:12951:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763779.323762:0:12951:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763779.323773:0:12951:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507846400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763779.323787:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507845760/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763779.323802:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763779.323806:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763779.323810:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507845760:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763779.323835:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763779.323883:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763779.323896:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffff8d25c8022a00 x1876699507845888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763779.323906:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507845888/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763779.323918:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763779.323920:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763779.323924:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507845888:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763779.323929:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763779.323944:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763779.323955:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffff8d25c8021180 x1876699507846016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763779.323968:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507846016/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763779.323986:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763779.323990:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763779.323994:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763779.324000:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846144:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763779.324018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763779.324026:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffff8d25c8020a80 x1876699507846144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763779.324041:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507846144/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763779.324047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763779.324049:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763779.324052:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846144:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763779.324056:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507846272:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763779.324069:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763779.324076:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffff8d25c8023800 x1876699507846272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763779.324083:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507846272/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763779.324088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763779.324090:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763779.324092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507846272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763779.324095:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507846400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763779.324106:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763779.324112:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763779/real 1789763779] req@ffff8d25c8022680 x1876699507846400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763779.324120:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507846400/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763804 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763779.324125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763779.324126:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763779.324128:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507846400:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763779.387532:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763780.411520:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763781.435554:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763782.459609:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763783.483514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763784.443489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763784.443501:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763784.443506:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763784.443519:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3762 00000100:00080000:3.0:1789763784.443531:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763784.443540:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763784.443601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507846528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763784.443633:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763784.443636:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763784.443637:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763784.443641:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3762 00000100:00080000:3.0:1789763784.443644:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763784.443647:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763784.443657:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763784.443668:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507846656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763784.443677:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:1.0:1789763784.443681:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507846528:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763784.443681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763784.443685:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763784.443693:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3762 00000100:00080000:3.0:1789763784.443696:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763784.443698:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763784.443701:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763784.443709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507846784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763784.443717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763784.443719:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763784.443720:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763784.443723:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3762 00000100:00080000:3.0:1789763784.443725:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763784.443727:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763784.443731:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763784.443738:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507846912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763784.443745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763784.443748:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763784.443749:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763784.443752:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3762 00000100:00080000:3.0:1789763784.443754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763784.443757:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763784.443759:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:1.0:1789763784.443766:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763784.443770:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507847040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763784.443778:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763784.443781:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763784.443782:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763784.443785:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3762 00000100:00080000:3.0:1789763784.443787:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763784.443790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763784.443793:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763784.443805:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507847168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763784.443819:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763784/real 1789763784] req@ffff8d25d1b09500 x1876699507846528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763784.443842:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507846528/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763784.443856:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763784.443861:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763784.443866:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507846528:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763784.443894:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846656:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763784.443914:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763784.443924:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763784/real 1789763784] req@ffff8d25d1b08380 x1876699507846656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763784.443933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507846656/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763784.443939:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763784.443942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763784.443944:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846656:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763784.443948:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763784.443960:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763784.443967:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763784/real 1789763784] req@ffff8d25d1b0b100 x1876699507846784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763784.443975:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507846784/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763784.443986:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763784.443989:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763784.443993:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763784.443999:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846912:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763784.444019:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763784.444031:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763784/real 1789763784] req@ffff8d25d1b0a300 x1876699507846912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763784.444044:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507846912/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763784.444050:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763784.444053:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763784.444055:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507846912:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763784.444059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763784.444074:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763784.444082:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763784/real 1789763784] req@ffff8d25d1b08000 x1876699507847040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763784.444090:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507847040/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763784.444095:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763784.444097:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763784.444099:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763784.444103:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763784.444114:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763784.444120:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763784/real 1789763784] req@ffff8d25d1b0b480 x1876699507847168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763784.444127:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507847168/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763809 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763784.444132:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763784.444134:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763784.444135:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847168:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763784.507529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763785.531523:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763786.555507:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763787.579491:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763788.603574:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763789.563611:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763789.563627:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763789.563633:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763789.563674:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3767 00000100:00080000:0.0:1789763789.563690:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763789.563705:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763789.563754:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507847296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763789.563787:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763789.563792:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763789.563794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763789.563800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3767 00000100:00080000:0.0:1789763789.563805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763789.563826:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763789.563837:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763789.563851:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507847296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763789.563857:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507847424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763789.563885:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763789.563890:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763789.563892:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763789.563895:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3767 00000100:00080000:0.0:1789763789.563898:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763789.563901:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763789.563907:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763789.563921:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507847552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763789.563937:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763789.563940:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763789.563942:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763789.563946:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3767 00000100:00080000:0.0:1789763789.563990:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763789.563994:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763789.563999:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:1.0:1789763789.564007:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763789.564014:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507847680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763789.564025:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763789.564029:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763789.564031:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763789.564034:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3767 00000100:00080000:0.0:1789763789.564042:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763789.564057:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763789.564061:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763789.564070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507847808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763789.564080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763789.564084:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763789.564087:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763789.564091:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3767 00000100:00080000:0.0:1789763789.564097:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:1.0:1789763789.564102:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763789/real 1789763789] req@ffff8d25d25b8700 x1876699507847296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763789.564108:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763789.564128:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763789.564149:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507847936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763789.564156:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507847296/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763789.564259:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763789.564267:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763789.564290:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507847296:192.168.203.141@tcp:250:kworker.0 00000100:00100000:1.0:1789763789.564340:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507847424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763789.564389:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763789.564462:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763789/real 1789763789] req@ffff8d25d25b8380 x1876699507847424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763789.564478:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507847424/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763789.564489:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763789.564493:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763789.564497:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507847424:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763789.564504:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507847552:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763789.564527:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763789.564539:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763789/real 1789763789] req@ffff8d25d25b8e00 x1876699507847552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763789.564552:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507847552/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763789.564561:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763789.564564:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763789.564567:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507847552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763789.564573:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507847680:192.168.203.141@tcp:8:kworker.0 00000800:00000100:1.0:1789763789.564591:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763789.564601:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763789/real 1789763789] req@ffff8d25d25bb100 x1876699507847680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763789.564613:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507847680/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763789.564622:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763789.564626:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763789.564629:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507847680:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763789.564635:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763789.564669:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763789.564678:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763789/real 1789763789] req@ffff8d25d25ba680 x1876699507847808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763789.564688:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507847808/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763789.564695:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763789.564697:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763789.564700:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763789.564705:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847936:192.168.203.141@tcp:38:kworker.0 00000800:00000100:1.0:1789763789.564719:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763789.564727:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763789/real 1789763789] req@ffff8d25d25b9c00 x1876699507847936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763789.564736:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507847936/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763814 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763789.564743:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763789.564745:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763789.564749:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507847936:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763789.627544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763790.651570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763791.675511:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763792.699501:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763793.723529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763794.683510:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763794.683522:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763794.683528:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763794.683542:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3772 00000100:00080000:3.0:1789763794.683553:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763794.683562:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763794.683601:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507848064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763794.683626:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763794.683629:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763794.683631:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763794.683634:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3772 00000100:00080000:3.0:1789763794.683638:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763794.683642:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763794.683650:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763794.683660:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507848192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763794.683672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763794.683675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763794.683676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:1.0:1789763794.683683:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507848064:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763794.683684:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3772 00000100:00080000:3.0:1789763794.683689:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763794.683696:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763794.683700:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763794.683709:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507848320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763794.683717:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763794.683720:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763794.683721:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763794.683724:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3772 00000100:00080000:3.0:1789763794.683727:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763794.683729:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763794.683733:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763794.683757:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507848448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763794.683764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763794.683767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763794.683768:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763794.683770:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3772 00000100:00080000:3.0:1789763794.683773:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763794.683775:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763794.683778:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763794.683785:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507848576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763794.683791:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763794.683793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763794.683794:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763794.683797:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3772 00000100:00080000:3.0:1789763794.683800:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763794.683819:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763794.683825:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763794.683833:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763794.683840:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507848704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:1.0:1789763794.683878:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763794.683899:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763794.683917:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507848576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763794.683935:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507848704:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763794.747557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763795.771544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763796.795508:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:0.0:1789763797.755596:0:2406:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:0.0:1789763797.755630:0:2406:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:0.0:1789763797.755642:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763797.755673:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763797.755682:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763797.755690:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763797.755698:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:0.0:1789763797.755708:0:2406:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:1.0:1789763797.755729:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763794/real 1789763797] req@ffff8d25d1b0b800 x1876699507848704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763797.755757:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507848704/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763797.755768:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763797.755771:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763797.755774:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507848704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763797.755780:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763794/real 1789763797] req@ffff8d25d1b0ad80 x1876699507848192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763797.755789:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507848192/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763797.755801:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763797.755806:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763797.755810:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848192:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763797.755817:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763794/real 1789763797] req@ffff8d25d1b09180 x1876699507848320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763797.755830:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507848320/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763797.755837:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763797.755841:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763797.755844:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763797.755851:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763794/real 1789763797] req@ffff8d25d1b0aa00 x1876699507848448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763797.755861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507848448/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763797.755869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763797.755873:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763797.755876:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848448:192.168.203.141@tcp:8:kworker.0 00000100:00100000:1.0:1789763797.755883:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763794/real 1789763797] req@ffff8d25d1b09c00 x1876699507848576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763797.755895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507848576/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763797.755904:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763797.755907:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763797.755910:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507848576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:1.0:1789763797.755916:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763794/real 1789763797] req@ffff8d25d1b08a80 x1876699507848064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763797.755928:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507848064/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763819 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763797.755936:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:1.0:1789763797.755939:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:1.0:1789763797.755943:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507848064:192.168.203.141@tcp:250:kworker.0 00010000:00010000:2.0:1789763797.819528:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763798.843524:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763799.803479:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763799.803489:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763799.803493:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763799.803503:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3777 00000100:00080000:1.0:1789763799.803512:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763799.803519:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763799.803548:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a680 x1876699507848832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763799.803578:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763799.803582:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763799.803584:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763799.803589:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3777 00000100:00080000:1.0:1789763799.803593:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763799.803598:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763799.803609:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763799.803624:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816aa00 x1876699507848960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763799.803626:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507848832:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763799.803640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763799.803642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763799.803644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763799.803647:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3777 00000100:00080000:1.0:1789763799.803649:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763799.803652:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763799.803656:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763799.803665:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169500 x1876699507849088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763799.803672:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763799.803675:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763799.803676:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763799.803678:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3777 00000100:00080000:1.0:1789763799.803680:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763799.803682:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763799.803686:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763799.803694:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168000 x1876699507849216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763799.803699:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763799.803711:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763799.803715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763799.803717:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763799.803722:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3777 00000100:00080000:1.0:1789763799.803726:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763799.803730:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763799.803735:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763799.803749:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169f80 x1876699507849344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763799.803756:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763799/real 1789763799] req@ffff8d25c816a680 x1876699507848832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763799.803763:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763799.803773:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763799.803778:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763799.803784:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3777 00000100:00080000:1.0:1789763799.803788:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763799.803789:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a680 x1876699507848832/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:1.0:1789763799.803797:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763799.803807:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763799.803823:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:1.0:1789763799.803826:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168e00 x1876699507849472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763799.803831:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763799.803840:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507848832:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763799.803870:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763799.803897:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763799.803911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763799/real 1789763799] req@ffff8d25c816aa00 x1876699507848960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763799.803923:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816aa00 x1876699507848960/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763799.803931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763799.803934:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763799.803937:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507848960:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763799.803961:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849088:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763799.803983:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763799.803994:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763799/real 1789763799] req@ffff8d25c8169500 x1876699507849088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763799.804005:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169500 x1876699507849088/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763799.804013:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763799.804017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763799.804020:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763799.804026:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849216:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763799.804045:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763799.804057:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763799/real 1789763799] req@ffff8d25c8168000 x1876699507849216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763799.804068:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168000 x1876699507849216/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763799.804076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763799.804080:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763799.804083:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849216:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763799.804090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507849344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763799.804122:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763799.804134:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763799/real 1789763799] req@ffff8d25c8169f80 x1876699507849344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763799.804147:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169f80 x1876699507849344/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763799.804155:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763799.804159:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763799.804162:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507849344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763799.804170:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507849472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763799.804968:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763799.804985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763799/real 1789763799] req@ffff8d25c8168e00 x1876699507849472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763799.805001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168e00 x1876699507849472/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763824 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763799.805010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763799.805012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763799.805016:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507849472:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763799.867546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763800.891557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763801.915565:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763802.939545:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763803.963522:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763804.923501:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763804.923509:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763804.923513:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763804.923521:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3782 00000100:00080000:0.0:1789763804.923529:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763804.923535:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763804.923583:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507849600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763804.923609:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763804.923613:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763804.923615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763804.923620:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3782 00000100:00080000:0.0:1789763804.923625:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763804.923629:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763804.923637:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507849600:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789763804.923640:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763804.923667:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507849728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763804.923680:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763804.923684:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763804.923686:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763804.923693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763804.923694:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3782 00000100:00080000:0.0:1789763804.923701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763804.923705:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763804.923710:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763804.923722:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507849856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763804.923729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763804.923734:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00100000:3.0:1789763804.923734:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763804/real 1789763804] req@ffff8d25d25bb800 x1876699507849600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763804.923738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763804.923745:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3782 00000100:00080000:0.0:1789763804.923750:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763804.923753:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763804.923756:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763804.923758:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507849600/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763804.923767:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507849984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763804.923768:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763804.923776:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763804.923780:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507849600:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763804.923794:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763804.923797:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763804.923799:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763804.923802:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849728:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763804.923806:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3782 00000100:00080000:0.0:1789763804.923810:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763804.923814:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763804.923817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763804.923827:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763804.923829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507850112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763804.923839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763804.923840:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763804/real 1789763804] req@ffff8d25d25b9f80 x1876699507849728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763804.923848:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763804.923852:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763804.923857:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507849728/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763804.923858:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3782 00000100:00080000:0.0:1789763804.923864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763804.923871:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763804.923877:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763804.923886:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763804.923891:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507850240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763804.923893:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763804.923900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849728:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763804.923912:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849856:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763804.923927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763804.923934:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763804/real 1789763804] req@ffff8d25d25bb480 x1876699507849856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763804.923942:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507849856/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763804.923947:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763804.923950:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763804.923952:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763804.923955:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849984:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763804.923964:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763804.923976:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763804/real 1789763804] req@ffff8d25d25bbb80 x1876699507849984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763804.923983:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507849984/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763804.923989:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763804.923991:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763804.923993:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507849984:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763804.924003:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507850112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763804.924014:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763804.924021:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763804/real 1789763804] req@ffff8d25d25b9180 x1876699507850112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763804.924028:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507850112/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763804.924033:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763804.924034:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763804.924036:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507850112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763804.924040:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507850240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763804.924049:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763804.924055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763804/real 1789763804] req@ffff8d25d25b8a80 x1876699507850240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763804.924063:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507850240/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763829 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763804.924067:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763804.924069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763804.924071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507850240:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763804.987500:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763806.011474:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763807.035543:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763808.059525:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763809.083502:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763810.043567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763810.043580:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763810.043587:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763810.043607:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3787 00000100:00080000:3.0:1789763810.043621:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763810.043639:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763810.043698:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507850368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763810.043738:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763810.043743:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763810.043746:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763810.043753:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3787 00000100:00080000:3.0:1789763810.043761:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763810.043767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763810.043781:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763810.043803:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507850368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763810.043829:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507850496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763810.043851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763810.043859:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763810.043863:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789763810.043869:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763810.043878:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3787 00000100:00080000:3.0:1789763810.043889:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763810.043916:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763810.043929:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763810.043937:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffff8d25d1b08700 x1876699507850368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763810.043950:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507850624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763810.043962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507850368/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763810.043996:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763810.044000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763810.044003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507850368:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763810.044024:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507850496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763810.044041:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763810.044050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffff8d25d1b08e00 x1876699507850496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763810.044061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507850496/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763810.044071:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763810.044076:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763810.044080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507850496:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763810.044327:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763810.044331:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763810.044334:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763810.044342:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3787 00000100:00100000:0.0:1789763810.044367:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507850624:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763810.044426:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763810.044452:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763810.044454:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffff8d25d1b09880 x1876699507850624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763810.044489:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763810.044497:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507850624/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763810.044502:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:0.0:1789763810.044522:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763810.044527:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763810.044533:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507850624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763810.044540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507850752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763810.044566:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763810.044571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763810.044574:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763810.044581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3787 00000100:00080000:3.0:1789763810.044589:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763810.044593:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507850752:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789763810.044597:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763810.044612:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763810.044633:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507850880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763810.044641:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763810.044660:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffff8d25d1b0bb80 x1876699507850752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763810.044661:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763810.044681:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763810.044689:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763810.044710:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3787 00000100:00100000:0.0:1789763810.044716:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507850752/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763810.044721:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763810.044736:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763810.044737:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763810.044759:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763810.044773:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507850752:192.168.203.141@tcp:8:kworker.0 00000008:00100000:3.0:1789763810.044779:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763810.044789:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507850880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763810.044804:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507851008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763810.044812:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763810.044827:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffff8d25d1b0a680 x1876699507850880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763810.044849:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507850880/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763810.044857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763810.044860:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763810.044863:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507850880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763810.044891:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507851008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763810.044904:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763810.044911:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763810/real 1789763810] req@ffff8d25d1b0bb80 x1876699507851008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763810.044918:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507851008/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763835 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763810.044923:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763810.044925:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763810.044927:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507851008:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763810.107573:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763811.131475:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763812.155573:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763813.179546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763814.203606:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763815.163597:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763815.163610:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763815.163615:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763815.163655:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3792 00000100:00080000:3.0:1789763815.163666:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763815.163674:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763815.163745:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507851136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763815.163790:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763815.163794:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763815.163796:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763815.163800:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3792 00000100:00080000:3.0:1789763815.163805:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763815.163811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763815.163821:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763815.163831:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507851136:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763815.163839:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507851264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763815.163851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763815.163855:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763815.163857:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763815.163860:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3792 00000100:00080000:3.0:1789763815.163863:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763815.163867:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763815.163872:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763815.163882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507851392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763815.163891:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763815.163894:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763815.163896:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763815.163900:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3792 00000800:00000100:0.0:1789763815.163906:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763815.163908:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763815.163914:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763815.163920:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763815.163930:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507851520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763815.163947:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763815.163951:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763815.163953:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763815.163957:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3792 00000100:00080000:3.0:1789763815.163960:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763815.163964:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763815.163969:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763815] req@ffff8d25d1b09880 x1876699507851136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789763815.163972:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763815.163990:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507851648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763815.163998:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507851136/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763815.164007:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763815.164018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763815.164023:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763815.164027:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763815.164030:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3792 00000100:00080000:0.0:1789763815.164036:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763815.164037:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763815.164046:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507851136:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763815.164047:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763815.164056:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763815.164070:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507851776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763815.164083:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507851264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763815.164128:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763815.164160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763815] req@ffff8d25d1b08e00 x1876699507851264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763815.164170:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507851264/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763815.164177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763815.164179:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763815.164182:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507851264:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763815.164186:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507851392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763815.164207:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763815.164221:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763815] req@ffff8d25d1b08700 x1876699507851392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763815.164236:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507851392/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763815.164245:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763815.164249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763815.164253:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507851392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763815.164257:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507851520:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763815.164274:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763815.164283:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763815] req@ffff8d25d1b08a80 x1876699507851520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763815.164297:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507851520/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763815.164302:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763815.164304:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763815.164307:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507851520:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763815.164310:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507851648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763815.164321:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763815.164328:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763815] req@ffff8d25d1b09c00 x1876699507851648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763815.164336:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507851648/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763815.164341:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763815.164343:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763815.164345:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507851648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763815.164381:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507851776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763815.164397:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763815.164406:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763815/real 1789763815] req@ffff8d25d1b0aa00 x1876699507851776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763815.164486:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507851776/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763840 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763815.164498:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763815.164501:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763815.164504:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507851776:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763815.227638:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763816.251550:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763817.275644:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763818.299529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763819.323631:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763820.283467:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763820.283474:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763820.283478:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763820.283486:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3797 00000100:00080000:3.0:1789763820.283495:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763820.283503:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763820.283540:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507851904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763820.283567:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763820.283571:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763820.283572:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763820.283577:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3797 00000100:00080000:3.0:1789763820.283581:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763820.283586:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763820.283596:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763820.283606:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507851904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763820.283613:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507852032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763820.283622:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763820.283625:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763820.283626:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763820.283629:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3797 00000100:00080000:3.0:1789763820.283632:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763820.283635:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763820.283639:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763820.283647:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507852160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763820.283653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763820.283656:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763820.283657:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763820.283659:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3797 00000100:00080000:3.0:1789763820.283661:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763820.283665:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789763820.283669:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763820.283671:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763820.283681:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507852288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763820.283692:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763820.283694:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763820.283696:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763820.283699:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3797 00000100:00080000:3.0:1789763820.283701:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763820.283704:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763820.283707:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763820.283709:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffff8d25d1b09180 x1876699507851904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763820.283717:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507852416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763820.283726:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763820.283731:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763820.283732:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507851904/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763820.283733:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763820.283741:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3797 00000100:00080000:3.0:1789763820.283746:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763820.283750:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763820.283752:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763820.283754:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763820.283758:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763820.283762:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507851904:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763820.283765:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507852544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763820.283786:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763820.283804:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763820.283813:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffff8d25d1b0ad80 x1876699507852032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763820.283821:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507852032/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763820.283826:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763820.283829:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763820.283831:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852032:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763820.283835:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852160:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763820.283846:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763820.283852:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffff8d25d1b0b800 x1876699507852160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763820.283859:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507852160/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763820.283865:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763820.283867:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763820.283869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763820.283872:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852288:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763820.283883:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763820.283889:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffff8d25d1b0b480 x1876699507852288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763820.283896:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507852288/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763820.283900:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763820.283902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763820.283904:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852288:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763820.283907:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507852416:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763820.283917:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763820.283923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffff8d25d1b08000 x1876699507852416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763820.283931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507852416/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763820.283936:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763820.283937:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763820.283939:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507852416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763820.283942:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507852544:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763820.283951:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763820.283958:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763820/real 1789763820] req@ffff8d25d1b0a300 x1876699507852544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763820.283965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507852544/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763845 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763820.283970:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763820.283972:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763820.283974:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507852544:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763820.347549:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763821.371632:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763822.395587:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763823.419551:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763824.443503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763825.403542:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763825.403553:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763825.403558:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763825.403568:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3803 00000100:00080000:1.0:1789763825.403588:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763825.403597:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763825.403636:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168380 x1876699507852672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763825.403662:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763825.403664:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763825.403666:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763825.403669:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3803 00000100:00080000:1.0:1789763825.403671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763825.403674:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763825.403682:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763825.403691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168700 x1876699507852800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763825.403699:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763825.403703:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763825.403704:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507852672:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763825.403705:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763825.403713:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3803 00000100:00080000:1.0:1789763825.403715:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763825.403718:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763825.403721:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763825.403730:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507852928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763825.403737:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763825.403740:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763825.403741:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763825.403743:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3803 00000100:00080000:1.0:1789763825.403757:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763825.403760:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763825.403764:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789763825.403767:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:1.0:1789763825.403775:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507853056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763825.403784:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763825.403787:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763825.403788:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763825.403790:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3803 00000100:00080000:1.0:1789763825.403793:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763825.403795:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763825.403799:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763825.403808:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763825/real 1789763825] req@ffff8d25c8168380 x1876699507852672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763825.403809:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507853184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763825.403834:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763825.403840:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763825.403843:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763825.403849:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3803 00000100:00080000:1.0:1789763825.403853:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763825.403856:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763825.403860:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763825.403865:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168380 x1876699507852672/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763825.403868:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507853312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763825.403894:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763825.403902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763825.403907:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507852672:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763825.403926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763825.403942:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763825.403950:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763825/real 1789763825] req@ffff8d25c8168700 x1876699507852800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763825.403959:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168700 x1876699507852800/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763825.403965:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763825.403967:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763825.403969:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852800:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763825.403973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852928:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763825.403983:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763825.403989:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763825/real 1789763825] req@ffff8d25c8168a80 x1876699507852928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763825.403996:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507852928/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763825.404005:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763825.404007:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763825.404009:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507852928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763825.404013:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853056:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763825.404023:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763825.404043:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763825/real 1789763825] req@ffff8d25c816a300 x1876699507853056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763825.404056:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507853056/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763825.404065:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763825.404068:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763825.404071:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853056:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763825.404078:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507853184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763825.404098:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763825.404108:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763825/real 1789763825] req@ffff8d25c816ad80 x1876699507853184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763825.404117:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507853184/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763825.404125:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763825.404128:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763825.404132:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507853184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763825.404139:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507853312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763825.404159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763825.404172:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763825/real 1789763825] req@ffff8d25c8169c00 x1876699507853312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763825.404181:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507853312/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763850 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763825.404186:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763825.404188:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763825.404190:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507853312:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763825.467542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763826.491559:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763827.515550:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763828.539556:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763829.563570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:1.0:1789763830.523454:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763830.523461:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:1.0:1789763830.523465:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763830.523473:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3808 00000100:00080000:1.0:1789763830.523480:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763830.523486:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:1.0:1789763830.523516:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816b480 x1876699507853440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763830.523536:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763830.523539:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763830.523541:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763830.523545:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3808 00000100:00080000:1.0:1789763830.523547:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763830.523550:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763830.523558:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763830.523566:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169880 x1876699507853568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763830.523574:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763830.523576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763830.523578:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763830.523581:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3808 00000100:00080000:1.0:1789763830.523583:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763830.523587:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507853440:192.168.203.141@tcp:250:kworker.0 00000100:00080000:1.0:1789763830.523588:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763830.523596:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763830.523606:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8169c00 x1876699507853696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763830.523613:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763830.523616:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:1.0:1789763830.523617:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763830.523620:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3808 00000100:00080000:1.0:1789763830.523622:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763830.523624:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763830.523628:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:1.0:1789763830.523634:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816ad80 x1876699507853824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763830.523640:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763830.523643:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:1.0:1789763830.523644:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763830.523646:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3808 00000800:00000100:0.0:1789763830.523651:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:1.0:1789763830.523653:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763830.523658:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763830.523662:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:1.0:1789763830.523669:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c816a300 x1876699507853952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:1.0:1789763830.523676:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:1.0:1789763830.523678:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:1.0:1789763830.523680:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:1.0:1789763830.523682:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3808 00000100:00080000:1.0:1789763830.523685:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:1.0:1789763830.523688:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:1.0:1789763830.523691:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763830.523702:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763830/real 1789763830] req@ffff8d25c816b480 x1876699507853440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:1.0:1789763830.523702:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8168a80 x1876699507854080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763830.523737:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816b480 x1876699507853440/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763830.523746:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763830.523749:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763830.523752:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507853440:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763830.523758:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763830.523776:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763830.523786:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763830/real 1789763830] req@ffff8d25c8169880 x1876699507853568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763830.523794:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169880 x1876699507853568/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763830.523799:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763830.523802:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763830.523804:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853568:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763830.523842:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853696:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763830.523858:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763830.523867:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763830/real 1789763830] req@ffff8d25c8169c00 x1876699507853696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763830.523892:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8169c00 x1876699507853696/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763830.523897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763830.523899:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763830.523901:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8169c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763830.523905:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853824:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763830.523915:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763830.523921:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763830/real 1789763830] req@ffff8d25c816ad80 x1876699507853824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763830.523933:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816ad80 x1876699507853824/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763830.523954:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763830.523958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763830.523961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507853824:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763830.523968:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507853952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763830.523990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763830.524043:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763830/real 1789763830] req@ffff8d25c816a300 x1876699507853952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763830.524052:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c816a300 x1876699507853952/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763830.524057:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763830.524059:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763830.524061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c816a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507853952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763830.524064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507854080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763830.524077:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763830.524083:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763830/real 1789763830] req@ffff8d25c8168a80 x1876699507854080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763830.524090:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8168a80 x1876699507854080/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763855 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763830.524100:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763830.524102:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763830.524104:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8168a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507854080:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763830.587503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763831.611632:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763832.635599:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763833.659599:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763834.683609:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763835.643592:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763835.643604:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763835.643610:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763835.643623:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3813 00000100:00080000:0.0:1789763835.643635:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763835.643647:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763835.643703:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507854208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763835.643756:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763835.643764:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763835.643769:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763835.643776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3813 00000100:00080000:0.0:1789763835.643783:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763835.643790:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763835.643804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763835.643814:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507854208:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763835.643831:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507854336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763835.643849:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763835.643856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763835.643859:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763835.643864:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3813 00000100:00080000:0.0:1789763835.643871:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763835.643875:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763835.643882:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763835.643905:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763835.643909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507854464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763835.643935:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763835.643942:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763835.643946:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763835.643983:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3813 00000100:00080000:0.0:1789763835.643993:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763835.644001:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763835/real 1789763835] req@ffff8d25d25bad80 x1876699507854208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763835.644010:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763835.644031:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763835.644062:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507854592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763835.644078:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507854208/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763835.644092:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763835.644109:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763835.644116:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763835.644131:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3813 00000100:00080000:3.0:1789763835.644138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763835.644139:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763835.644150:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789763835.644158:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:0.0:1789763835.644163:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763835.644170:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507854208:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763835.644184:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507854720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763835.644198:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763835.644203:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763835.644206:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763835.644216:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507854336:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763835.644217:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3813 00000100:00080000:0.0:1789763835.644227:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763835.644234:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763835.644241:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763835.644261:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507854848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763835.644276:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763835.644305:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763835/real 1789763835] req@ffff8d25d25ba300 x1876699507854336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763835.644324:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507854336/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763835.644334:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763835.644338:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763835.644343:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507854336:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763835.644353:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507854464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763835.644434:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763835.644459:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763835/real 1789763835] req@ffff8d25d25b8000 x1876699507854464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763835.644480:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507854464/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763835.644499:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763835.644503:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763835.644507:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507854464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763835.644515:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507854592:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763835.644543:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763835.644564:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763835/real 1789763835] req@ffff8d25d25b8a80 x1876699507854592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763835.644587:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507854592/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763835.644605:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763835.644611:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763835.644618:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507854592:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763835.644631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507854720:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763835.644669:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763835.644687:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763835/real 1789763835] req@ffff8d25d25b9180 x1876699507854720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763835.644702:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507854720/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763835.644712:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763835.644716:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763835.644721:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507854720:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763835.644745:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507854848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763835.644768:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763835.644781:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763835/real 1789763835] req@ffff8d25d25bbb80 x1876699507854848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763835.644796:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507854848/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763860 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763835.644806:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763835.644810:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763835.644814:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507854848:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763835.707554:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763836.731649:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763837.755581:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763838.779674:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763839.803625:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763840.763511:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763840.763529:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763840.763537:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763840.763555:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3818 00000100:00080000:0.0:1789763840.763572:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763840.763585:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763840.763653:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507854976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763840.763698:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763840.763704:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763840.763707:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763840.763714:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3818 00000100:00080000:0.0:1789763840.763719:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763840.763726:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763840.763743:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763840.763763:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507854976:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763840.763766:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507855104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763840.763789:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763840.763793:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763840.763795:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763840.763801:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3818 00000100:00080000:0.0:1789763840.763806:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763840.763811:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763840.763817:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763840.763834:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507855232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763840.763847:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763840.763852:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763840.763854:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763840.763859:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3818 00000100:00080000:0.0:1789763840.763864:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763840.763869:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763840.763875:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763840.763885:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763840.763902:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507855360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763840.763951:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763840.763957:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763840.763961:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763840.763969:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3818 00000100:00080000:0.0:1789763840.763977:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763840.763985:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763840/real 1789763840] req@ffff8d25d25bb480 x1876699507854976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763840.763987:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763840.764003:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763840.764031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507854976/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763840.764034:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507855488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763840.764076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763840.764080:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763840.764091:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763840.764091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763840.764100:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763840.764104:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507854976:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763840.764113:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3818 00000100:00080000:0.0:1789763840.764122:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763840.764138:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763840.764145:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763840.764152:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763840.764175:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507855616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763840.764194:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763840.764224:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763840/real 1789763840] req@ffff8d25d25b9f80 x1876699507855104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763840.764246:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507855104/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763840.764258:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763840.764263:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763840.764268:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855104:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763840.764275:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763840.764297:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763840.764310:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763840/real 1789763840] req@ffff8d25d25bb800 x1876699507855232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763840.764324:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507855232/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763840.764334:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763840.764338:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763840.764341:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763840.764348:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855360:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763840.764368:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763840.764412:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763840/real 1789763840] req@ffff8d25d25b9c00 x1876699507855360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763840.764428:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507855360/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763840.764440:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763840.764444:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763840.764448:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855360:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763840.764454:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507855488:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763840.764478:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763840.764491:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763840/real 1789763840] req@ffff8d25d25ba680 x1876699507855488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763840.764505:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507855488/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763840.764515:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763840.764518:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763840.764521:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507855488:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763840.764543:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507855616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763840.764564:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763840.764577:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763840/real 1789763840] req@ffff8d25d25bb100 x1876699507855616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763840.764592:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507855616/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763865 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763840.764601:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763840.764605:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763840.764609:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507855616:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763840.827644:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763841.851623:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763842.875716:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763843.899631:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763844.923710:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763845.883581:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763845.883596:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763845.883604:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763845.883625:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3823 00000100:00080000:0.0:1789763845.883653:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763845.883672:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763845.883735:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507855744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763845.883777:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763845.883783:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763845.883785:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763845.883791:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3823 00000100:00080000:0.0:1789763845.883795:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763845.883801:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763845.883815:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763845.883837:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507855872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763845.883851:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763845.883856:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763845.883858:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763845.883867:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3823 00000100:00080000:0.0:1789763845.883874:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763845.883905:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763845.883909:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507855744:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789763845.883922:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763845.883955:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507856000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763845.883982:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763845.883988:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763845.883993:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763845.884004:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3823 00000800:00000100:3.0:1789763845.884009:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763845.884014:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763845.884022:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763845.884031:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763845.884049:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507856128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763845.884064:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763845.884073:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763845.884076:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763845.884081:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3823 00000100:00080000:0.0:1789763845.884085:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763845.884089:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763845.884095:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763845.884106:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507856256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763845.884117:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763845.884122:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763845.884124:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763845.884128:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3823 00000100:00080000:0.0:1789763845.884133:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763845.884137:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763845.884143:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763845.884153:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507856384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763845.884154:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763845/real 1789763845] req@ffff8d25d25b8e00 x1876699507855744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763845.884403:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507855744/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763845.884422:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763845.884429:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763845.884435:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507855744:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763845.884522:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855872:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763845.884564:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763845.884584:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763845/real 1789763845] req@ffff8d25d25b8380 x1876699507855872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763845.884598:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507855872/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763845.884608:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763845.884611:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763845.884615:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507855872:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763845.884641:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763845.884659:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763845.884670:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763845/real 1789763845] req@ffff8d25d25b8700 x1876699507856000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763845.884684:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507856000/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763845.884692:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763845.884696:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763845.884700:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763845.884706:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856128:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763845.884722:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763845.884731:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763845/real 1789763845] req@ffff8d25d25bb100 x1876699507856128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763845.884743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507856128/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763845.884752:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763845.884755:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763845.884759:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856128:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763845.884765:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507856256:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763845.884780:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763845.884789:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763845/real 1789763845] req@ffff8d25d25ba680 x1876699507856256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763845.884801:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507856256/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763845.884811:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763845.884814:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763845.884818:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507856256:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763845.884824:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507856384:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763845.884839:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763845.884848:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763845/real 1789763845] req@ffff8d25d25b9c00 x1876699507856384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763845.884860:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507856384/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763870 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763845.884869:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763845.884872:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763845.884875:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507856384:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763845.947566:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763846.971600:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763847.995621:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763849.019623:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763850.043643:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763851.003624:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.003642:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763851.003650:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.003674:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3828 00000100:00080000:2.0:1789763851.003691:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.003703:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763851.003769:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507856512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763851.003814:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.003820:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763851.003822:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.003829:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3828 00000100:00080000:2.0:1789763851.003833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.003839:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763851.003885:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763851.003908:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507856640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763851.003926:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507856512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763851.003933:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.003960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763851.003966:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.003980:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3828 00000100:00080000:2.0:1789763851.003990:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.004002:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763851.004014:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763851.004034:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763851.004047:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507856768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763851.004075:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.004081:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763851.004085:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.004091:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3828 00000100:00080000:2.0:1789763851.004098:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.004102:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763851.004111:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763851.004145:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507856896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763851.004146:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffff8d25c8023b80 x1876699507856512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763851.004215:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.004227:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:3.0:1789763851.004228:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507856512/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763851.004239:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763851.004261:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3828 00000100:00080000:2.0:1789763851.004273:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763851.004275:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763851.004284:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763851.004293:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763851.004297:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763851.004314:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507857024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763851.004318:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507856512:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763851.004358:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763851.004367:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763851.004372:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763851.004434:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856640:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763851.004464:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3828 00000100:00080000:2.0:1789763851.004478:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763851.004495:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763851.004496:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763851.004518:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763851.004529:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffff8d25c8022300 x1876699507856640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.004554:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507856640/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763851.004570:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507857152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763851.004573:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.004590:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763851.004630:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856640:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763851.004651:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763851.004681:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763851.004697:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffff8d25c8020380 x1876699507856768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.004712:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507856768/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763851.004734:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.004738:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763851.004742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763851.004749:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856896:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763851.004769:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763851.004782:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffff8d25c8022d80 x1876699507856896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.004812:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507856896/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763851.004828:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.004836:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763851.004869:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507856896:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763851.004883:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857024:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763851.004926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763851.004945:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffff8d25c8023100 x1876699507857024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.004962:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507857024/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763851.004972:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.004976:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763851.004980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857024:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763851.005006:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763851.005031:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763851.005044:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763851/real 1789763851] req@ffff8d25c8020e00 x1876699507857152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763851.005066:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507857152/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763876 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763851.005083:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763851.005089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763851.005096:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857152:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763851.067628:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763852.091766:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763853.115586:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763854.139612:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763855.163597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763856.123568:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763856.123593:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763856.123599:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763856.123612:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3833 00000100:00080000:0.0:1789763856.123624:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763856.123634:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763856.123699:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507857280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763856.123749:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763856.123754:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763856.123756:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763856.123762:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3833 00000100:00080000:0.0:1789763856.123767:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763856.123773:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763856.123786:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763856.123826:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507857280:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763856.123830:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507857408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763856.123866:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763856.123871:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763856.123874:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763856.123879:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3833 00000100:00080000:0.0:1789763856.123885:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763856.123891:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763856.123899:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763856.123917:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507857536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763856.123931:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:3.0:1789763856.123939:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763856.123942:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763856.123949:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763856.123956:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3833 00000100:00080000:0.0:1789763856.123963:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763856.123968:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763856.123975:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763856.123992:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507857664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763856.124007:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763856.124008:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffff8d25d25bb800 x1876699507857280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763856.124021:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763856.124027:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763856.124043:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3833 00000100:00080000:0.0:1789763856.124053:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763856.124060:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763856.124067:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763856.124069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507857280/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763856.124087:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507857792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763856.124089:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763856.124105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763856.124122:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507857280:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763856.124125:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763856.124137:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763856.124141:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763856.124147:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3833 00000100:00080000:0.0:1789763856.124153:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763856.124157:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763856.124163:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763856.124170:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507857408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763856.124181:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507857920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763856.124212:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763856.124237:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffff8d25d25b9f80 x1876699507857408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763856.124264:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507857408/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763856.124281:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763856.124288:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763856.124295:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507857408:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763856.124320:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507857536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763856.124361:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763856.124412:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffff8d25d25bb480 x1876699507857536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763856.124430:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507857536/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763856.124440:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763856.124444:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763856.124448:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507857536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763856.124456:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507857664:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763856.124483:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763856.124496:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffff8d25d25bbb80 x1876699507857664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763856.124510:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507857664/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763856.124520:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763856.124523:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763856.124527:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507857664:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763856.124534:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857792:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763856.124553:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763856.124573:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffff8d25d25b9180 x1876699507857792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763856.124588:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507857792/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763856.124597:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763856.124601:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763856.124604:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857792:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763856.124626:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763856.124645:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763856.124657:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763856/real 1789763856] req@ffff8d25d25b8a80 x1876699507857920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763856.124670:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507857920/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763881 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763856.124680:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763856.124692:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763856.124696:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507857920:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763856.187649:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763857.211608:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763858.235643:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763859.259601:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763860.283594:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763861.243612:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763861.243625:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763861.243631:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763861.243644:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3838 00000100:00080000:2.0:1789763861.243656:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763861.243665:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763861.243714:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507858048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763861.243791:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763861.243797:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763861.243799:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763861.243805:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3838 00000100:00080000:2.0:1789763861.243810:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763861.243815:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763861.243828:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763861.243854:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507858176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763861.243868:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763861.243871:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763861.243874:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763861.243878:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3838 00000100:00080000:2.0:1789763861.243881:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763861.243885:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763861.243890:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763861.243900:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507858304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763861.243910:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763861.243914:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763861.243916:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763861.243920:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3838 00000100:00080000:2.0:1789763861.243924:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763861.243928:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763861.243932:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763861.243941:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507858432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763861.243962:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763861.243966:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763861.243968:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763861.243973:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3838 00000100:00080000:2.0:1789763861.243976:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763861.243988:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763861.243993:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763861.244003:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507858560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763861.244016:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763861.244021:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763861.244023:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763861.244037:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3838 00000100:00080000:2.0:1789763861.244041:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763861.244045:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763861.244052:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763861.244064:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507858048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763861.244072:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507858688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763861.244230:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763861.244284:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763861.244316:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763861.244340:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507858560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763861.244429:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507858688:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763861.307596:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763862.331606:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763863.355646:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763864.315791:0:2403:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789763864.315843:0:2403:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763864.315866:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763864.315937:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763864.315973:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763864.315995:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789763864.316012:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763864] req@ffff8d25c8021500 x1876699507858048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1789763864.316015:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763864.316047:0:2403:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789763864.316074:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507858048/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763864.316090:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763864.316096:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763864.316102:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507858048:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763864.316111:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763864] req@ffff8d25c8023480 x1876699507858176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763864.316127:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507858176/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763864.316138:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763864.316141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763864.316146:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858176:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763864.316153:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763864] req@ffff8d25c8021c00 x1876699507858304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763864.316167:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507858304/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763864.316183:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763864.316187:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763864.316192:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763864.316205:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763864] req@ffff8d25c8021880 x1876699507858432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763864.316228:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507858432/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763864.316244:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763864.316249:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763864.316256:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858432:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763864.316269:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763864] req@ffff8d25c8020700 x1876699507858560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763864.316291:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507858560/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763864.316305:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763864.316310:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763864.316316:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507858560:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763864.316438:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763861/real 1789763864] req@ffff8d25c8020e00 x1876699507858688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763864.316465:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507858688/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763886 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763864.316482:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763864.316488:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763864.316496:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507858688:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763864.379596:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763865.403629:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763866.363494:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763866.363507:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763866.363513:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763866.363527:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3844 00000100:00080000:2.0:1789763866.363539:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763866.363550:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763866.363595:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507858816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763866.363630:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763866.363635:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763866.363638:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763866.363643:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3844 00000100:00080000:2.0:1789763866.363647:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763866.363652:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763866.363664:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763866.363680:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507858944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763866.363697:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763866.363704:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507858816:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763866.363705:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763866.363712:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763866.363726:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3844 00000100:00080000:2.0:1789763866.363752:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763866.363757:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763866.363763:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763866.363779:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507859072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763866.363802:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763866.363807:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763866.363810:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763866.363815:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3844 00000100:00080000:2.0:1789763866.363822:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763866.363826:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763866.363833:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763866.363838:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763866.363853:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507859200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763866.363872:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763866.363877:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763866.363879:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763866.363883:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3844 00000100:00080000:2.0:1789763866.363887:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763866.363892:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763866.363899:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763866.363928:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffff8d25c8023100 x1876699507858816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763866.363935:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507859328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763866.363966:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763866.363972:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763866.363975:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763866.363981:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507858816/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763866.363988:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3844 00000100:00080000:2.0:1789763866.364007:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763866.364015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763866.364019:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789763866.364024:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763866.364032:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507858816:192.168.203.141@tcp:250:kworker.0 00000008:00100000:2.0:1789763866.364033:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763866.364059:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507859456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763866.364069:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858944:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763866.364121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763866.364142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffff8d25c8022d80 x1876699507858944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.364164:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507858944/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.364177:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.364183:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763866.364189:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507858944:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763866.364198:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763866.364230:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763866.364248:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffff8d25c8020380 x1876699507859072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.364267:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507859072/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.364280:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.364285:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763866.364291:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763866.364300:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859200:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763866.364341:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763866.364359:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffff8d25c8022300 x1876699507859200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.364423:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507859200/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.364436:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.364441:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763866.364445:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859200:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763866.364455:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507859328:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763866.364511:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763866.364531:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffff8d25c8023b80 x1876699507859328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.364549:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507859328/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.364561:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.364565:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763866.364570:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507859328:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763866.364598:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507859456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763866.364630:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763866.364649:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763866/real 1789763866] req@ffff8d25c8022680 x1876699507859456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763866.364668:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507859456/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763891 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763866.364681:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763866.364686:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763866.364691:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507859456:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763866.427597:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763867.451605:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763868.475643:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763869.499735:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763870.523601:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763871.483533:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763871.483547:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763871.483554:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763871.483568:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3849 00000100:00080000:2.0:1789763871.483583:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763871.483594:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763871.483646:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507859584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763871.483686:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763871.483715:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763871.483719:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763871.483729:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3849 00000100:00080000:2.0:1789763871.483737:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763871.483747:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763871.483763:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763871.483787:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507859584:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763871.483801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507859712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763871.483830:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763871.483836:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763871.483840:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763871.483848:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3849 00000100:00080000:2.0:1789763871.483867:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763871.483879:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763871.483880:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763871.483892:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763871.483918:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507859840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763871.483939:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763871.483946:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763871.483951:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763871.483959:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3849 00000100:00080000:2.0:1789763871.483969:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763871.483979:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763871/real 1789763871] req@ffff8d25c8023800 x1876699507859584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763871.483984:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763871.484002:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763871.484032:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507859584/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763871.484038:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507859968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763871.484076:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763871.484082:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763871.484089:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763871.484093:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763871.484100:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763871.484104:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507859584:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763871.484112:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3849 00000100:00080000:2.0:1789763871.484122:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763871.484128:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763871.484135:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763871.484150:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763871.484154:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507860096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763871.484174:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763871.484179:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763871.484182:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763871.484188:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3849 00000100:00080000:2.0:1789763871.484207:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763871.484209:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763871.484218:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763871.484228:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763871.484248:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507860224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763871.484255:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763871/real 1789763871] req@ffff8d25c8020a80 x1876699507859712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763871.484291:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507859712/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763871.484305:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763871.484320:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763871.484325:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859712:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763871.484334:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763871.484373:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763871.484442:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763871/real 1789763871] req@ffff8d25c8021180 x1876699507859840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763871.484474:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507859840/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763871.484485:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763871.484489:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763871.484493:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763871.484501:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859968:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763871.484526:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763871.484540:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763871/real 1789763871] req@ffff8d25c8022a00 x1876699507859968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763871.484555:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507859968/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763871.484565:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763871.484569:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763871.484573:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507859968:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763871.484593:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860096:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763871.484614:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763871.484626:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763871/real 1789763871] req@ffff8d25c8020000 x1876699507860096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763871.484641:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507860096/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763871.484651:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763871.484655:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763871.484658:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860096:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763871.484665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763871.484683:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763871.484725:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763871/real 1789763871] req@ffff8d25c8023800 x1876699507860224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763871.484742:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507860224/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763896 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763871.484753:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763871.484756:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763871.484761:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860224:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763871.547537:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763872.571627:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763873.595623:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763874.619599:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763875.643626:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763876.603489:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763876.603504:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763876.603511:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763876.603527:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3854 00000100:00080000:0.0:1789763876.603553:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763876.603565:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763876.603618:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507860352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763876.603671:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763876.603679:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763876.603682:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763876.603687:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3854 00000100:00080000:0.0:1789763876.603693:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763876.603699:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763876.603715:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763876.603740:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507860352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763876.603746:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507860480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763876.603764:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763876.603768:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763876.603770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763876.603775:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3854 00000100:00080000:0.0:1789763876.603780:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763876.603784:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763876.603789:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763876.603801:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507860608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763876.603811:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763876.603817:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763876.603819:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763876.603824:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3854 00000100:00080000:0.0:1789763876.603827:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763876.603831:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763876.603836:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763876.603850:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507860736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763876.603861:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763876.603868:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763876.603873:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763876.603876:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763876.603882:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3854 00000100:00080000:0.0:1789763876.603888:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763876.603893:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763876.603898:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763876.603909:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507860864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763876.603920:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763876.603925:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763876.603929:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763876.603931:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763876/real 1789763876] req@ffff8d25d25b8000 x1876699507860352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763876.603937:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3854 00000100:00080000:0.0:1789763876.603945:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763876.603960:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763876.603967:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507860352/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763876.603970:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763876.603989:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507860992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763876.603992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763876.604009:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763876.604022:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507860352:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763876.604066:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507860480:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763876.604109:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763876.604124:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763876/real 1789763876] req@ffff8d25d25ba300 x1876699507860480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763876.604137:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507860480/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763876.604147:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763876.604151:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763876.604155:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507860480:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763876.604162:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507860608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763876.604179:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763876.604189:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763876/real 1789763876] req@ffff8d25d25bad80 x1876699507860608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763876.604201:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507860608/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763876.604209:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763876.604212:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763876.604215:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507860608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763876.604221:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507860736:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763876.604238:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763876.604247:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763876/real 1789763876] req@ffff8d25d25b8a80 x1876699507860736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763876.604259:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507860736/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763876.604267:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763876.604270:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763876.604273:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507860736:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763876.604278:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860864:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763876.604295:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763876.604304:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763876/real 1789763876] req@ffff8d25d25b9180 x1876699507860864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763876.604317:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507860864/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763876.604325:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763876.604328:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763876.604331:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860864:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763876.604336:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763876.604351:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763876.604360:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763876/real 1789763876] req@ffff8d25d25bbb80 x1876699507860992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763876.604415:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507860992/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763901 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763876.604424:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763876.604428:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763876.604432:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507860992:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763876.667596:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763877.691708:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763878.715544:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763879.739567:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763880.763592:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763881.723563:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763881.723576:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763881.723582:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763881.723596:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3859 00000100:00080000:0.0:1789763881.723610:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763881.723637:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763881.723689:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507861120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763881.723729:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763881.723735:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763881.723738:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763881.723744:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3859 00000100:00080000:0.0:1789763881.723751:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763881.723758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763881.723770:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763881.723794:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507861248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763881.723803:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507861120:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763881.723835:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763881.723842:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763881.723845:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763881.723855:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3859 00000100:00080000:0.0:1789763881.723862:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763881.723870:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763881.723880:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763881.723904:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507861376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763881.723919:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763881.723923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763881.723934:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763881.723935:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763881.723951:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3859 00000100:00080000:0.0:1789763881.723968:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763881.723976:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763881.723983:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763881.724000:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507861504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763881.724010:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763881.724014:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763881.724017:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763881.724022:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3859 00000100:00080000:0.0:1789763881.724026:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763881.724032:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763881.724036:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763881/real 1789763881] req@ffff8d25d25bb480 x1876699507861120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763881.724058:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763881.724075:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507861632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763881.724086:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507861120/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763881.724100:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763881.724125:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763881.724135:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763881.724161:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3859 00000100:00080000:0.0:1789763881.724176:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763881.724191:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:3.0:1789763881.724196:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:0.0:1789763881.724197:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763881.724216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763881.724223:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507861760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763881.724225:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507861120:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763881.724281:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507861248:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763881.724319:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763881.724348:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763881/real 1789763881] req@ffff8d25d25b9f80 x1876699507861248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763881.724452:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507861248/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763881.724464:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763881.724468:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763881.724488:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507861248:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763881.724496:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507861376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763881.724528:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763881.724562:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763881/real 1789763881] req@ffff8d25d25bb800 x1876699507861376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763881.724585:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507861376/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763881.724597:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763881.724603:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763881.724613:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507861376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763881.724659:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507861504:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763881.724709:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763881.724721:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763881/real 1789763881] req@ffff8d25d25b9c00 x1876699507861504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763881.724733:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507861504/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763881.724742:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763881.724746:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763881.724750:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507861504:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763881.724759:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507861632:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763881.724783:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763881.724893:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763881/real 1789763881] req@ffff8d25d25ba680 x1876699507861632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763881.724908:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507861632/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763881.724916:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763881.724920:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763881.724925:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507861632:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763881.724939:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507861760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763881.724961:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763881.724970:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763881/real 1789763881] req@ffff8d25d25bb100 x1876699507861760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763881.725161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507861760/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763906 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763881.725188:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763881.725194:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763881.725251:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507861760:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763881.787703:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763882.811517:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763883.835553:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763884.859651:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763885.883695:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763886.843540:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763886.843558:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763886.843566:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763886.843608:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3864 00000100:00080000:0.0:1789763886.843624:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763886.843655:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763886.843718:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507861888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763886.843758:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763886.843763:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763886.843766:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763886.843773:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3864 00000100:00080000:0.0:1789763886.843778:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763886.843785:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763886.843800:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763886.843817:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507862016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763886.843834:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507861888:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763886.843839:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763886.843862:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763886.843866:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763886.843873:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3864 00000100:00080000:0.0:1789763886.843879:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763886.843885:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763886.843891:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763886.843906:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507862144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763886.843918:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763886.843923:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763886.843927:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763886.843932:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3864 00000100:00080000:0.0:1789763886.843937:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763886.843947:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763886.843949:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763886.843972:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763886.843994:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507862272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763886.844014:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763886.844018:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763886.844021:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763886.844026:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3864 00000100:00080000:0.0:1789763886.844031:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763886.844038:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763886.844047:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763886.844062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763886/real 1789763886] req@ffff8d25d25b8700 x1876699507861888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763886.844066:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507862400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763886.844089:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763886.844113:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763886.844122:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763886.844148:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3864 00000100:00080000:0.0:1789763886.844156:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763886.844163:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763886.844176:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507861888/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763886.844178:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763886.844209:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507862528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763886.844212:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763886.844237:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763886.844258:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507861888:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763886.844300:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862016:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763886.844335:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763886.844353:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763886/real 1789763886] req@ffff8d25d25b8380 x1876699507862016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763886.844370:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507862016/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763886.844426:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763886.844431:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763886.844436:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862016:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763886.844444:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862144:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763886.844470:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763886.844484:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763886/real 1789763886] req@ffff8d25d25b8e00 x1876699507862144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763886.844499:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507862144/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763886.844509:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763886.844513:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763886.844517:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763886.844524:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862272:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763886.844544:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763886.844555:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763886/real 1789763886] req@ffff8d25d25bb100 x1876699507862272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763886.844569:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507862272/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763886.844611:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763886.844619:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763886.844643:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862272:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763886.844665:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507862400:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763886.844703:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763886.844722:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763886/real 1789763886] req@ffff8d25d25ba680 x1876699507862400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763886.844736:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507862400/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763886.844745:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763886.844748:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763886.844752:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507862400:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763886.844758:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507862528:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763886.844780:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763886.844790:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763886/real 1789763886] req@ffff8d25d25b9c00 x1876699507862528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763886.844802:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507862528/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763911 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763886.844810:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763886.844813:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763886.844817:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507862528:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763886.907722:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763887.931686:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763888.955543:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763889.979644:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763891.003596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763891.963529:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763891.963568:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763891.963577:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763891.963607:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3869 00000100:00080000:3.0:1789763891.963624:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763891.963636:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763891.963712:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507862656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763891.963762:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763891.963767:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763891.963770:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763891.963776:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3869 00000100:00080000:3.0:1789763891.963782:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763891.963788:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763891.963804:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763891.963824:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507862784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763891.963845:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507862656:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763891.963848:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763891.963887:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763891.963890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763891.963897:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3869 00000100:00080000:3.0:1789763891.963906:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763891.963912:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763891.963920:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763891.963938:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507862912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763891.963956:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763891.963960:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763891.963964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789763891.963976:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763891.963983:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3869 00000100:00080000:3.0:1789763891.963995:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763891.963999:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763891.964006:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763891.964029:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507863040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763891.964042:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763891.964047:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763891.964050:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763891.964054:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3869 00000100:00080000:3.0:1789763891.964060:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763891.964067:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763891.964076:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763891.964079:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763891/real 1789763891] req@ffff8d25d1b0b100 x1876699507862656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763891.964097:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507863168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763891.964150:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763891.964154:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763891.964157:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763891.964168:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3869 00000100:00100000:0.0:1789763891.964170:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507862656/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763891.964182:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763891.964203:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763891.964223:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:3.0:1789763891.964227:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789763891.964232:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763891.964245:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507862656:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763891.964264:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507863296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763891.964285:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862784:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763891.964327:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763891.964348:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763891/real 1789763891] req@ffff8d25d1b08380 x1876699507862784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763891.964365:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507862784/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763891.964423:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763891.964428:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763891.964433:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862784:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763891.964441:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763891.964468:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763891.964483:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763891/real 1789763891] req@ffff8d25d1b09500 x1876699507862912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763891.964507:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507862912/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763891.964526:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763891.964530:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763891.964534:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507862912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763891.964567:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863040:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763891.964610:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763891.964626:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763891/real 1789763891] req@ffff8d25d1b0a300 x1876699507863040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763891.964641:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507863040/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763891.964652:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763891.964656:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763891.964660:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863040:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763891.964668:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507863168:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763891.964688:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763891.964700:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763891/real 1789763891] req@ffff8d25d1b08000 x1876699507863168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763891.964725:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507863168/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763891.964734:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763891.964738:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763891.964742:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507863168:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763891.964772:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507863296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763891.964793:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763891.964806:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763891/real 1789763891] req@ffff8d25d1b0b480 x1876699507863296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763891.964821:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507863296/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763916 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763891.964831:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763891.964834:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763891.964838:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507863296:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763892.027656:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763893.051579:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763894.075588:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763895.099623:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763896.123632:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763897.083570:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763897.083588:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763897.083595:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763897.083618:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3874 00000100:00080000:0.0:1789763897.083638:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763897.083655:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763897.083736:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507863424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763897.083800:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763897.083808:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763897.083812:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763897.083823:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3874 00000100:00080000:0.0:1789763897.083833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763897.083843:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763897.083865:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763897.083871:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507863424:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763897.083934:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507863552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763897.083966:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763897.083976:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000800:00000100:3.0:1789763897.083979:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763897.083988:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763897.084003:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3874 00000100:00080000:0.0:1789763897.084018:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763897.084029:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763897.084044:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763897.084050:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffff8d25d25bb800 x1876699507863424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763897.084086:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507863680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763897.084087:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507863424/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763897.084172:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763897.084179:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763897.084185:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507863424:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763897.084219:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763897.084229:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763897.084235:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763897.084238:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863552:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763897.084261:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3874 00000100:00080000:0.0:1789763897.084276:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763897.084294:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763897.084295:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763897.084320:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763897.084325:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffff8d25d25b9f80 x1876699507863552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763897.084350:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507863552/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763897.084365:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507863808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763897.084369:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763897.084413:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763897.084419:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863552:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763897.084428:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763897.084471:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763897.084495:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffff8d25d25bb480 x1876699507863680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763897.084538:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763897.084547:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507863680/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763897.084564:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763897.084573:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763897.084578:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763897.084582:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763897.084598:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863680:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763897.084610:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3874 00000100:00080000:0.0:1789763897.084624:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763897.084632:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863808:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1789763897.084640:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763897.084661:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763897.084673:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763897.084692:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffff8d25d25bbb80 x1876699507863808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763897.084704:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507863936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763897.084718:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507863808/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763897.084777:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763897.084791:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763897.084793:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763897.084799:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507863808:192.168.203.141@tcp:8:kworker.0 00000100:00080000:0.0:1789763897.084823:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763897.084830:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763897.084837:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507863936:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763897.084850:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3874 00000100:00080000:0.0:1789763897.084864:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:3.0:1789763897.084875:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763897.084878:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763897.084895:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffff8d25d25b9180 x1876699507863936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763897.084896:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763897.084926:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507863936/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763897.084937:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763897.084947:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507864064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763897.084948:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763897.084961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507863936:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763897.085039:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507864064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763897.085073:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763897.085089:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763897/real 1789763897] req@ffff8d25d25b8a80 x1876699507864064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763897.085104:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507864064/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763922 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763897.085115:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763897.085119:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763897.085123:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507864064:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763897.147636:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763898.171629:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763899.195703:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763900.219634:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763901.243617:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763902.203594:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763902.203605:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763902.203611:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763902.203624:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3879 00000100:00080000:3.0:1789763902.203637:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763902.203648:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763902.203691:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507864192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763902.203727:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763902.203741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763902.203744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763902.203750:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3879 00000100:00080000:3.0:1789763902.203754:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763902.203760:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763902.203772:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763902.203788:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507864320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763902.203804:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507864192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763902.203808:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763902.203822:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763902.203824:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763902.203830:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3879 00000100:00080000:3.0:1789763902.203833:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763902.203838:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763902.203845:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763902.203861:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507864448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763902.203874:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763902.203878:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763902.203880:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763902.203886:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3879 00000100:00080000:3.0:1789763902.203891:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763902.203896:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789763902.203904:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763902.203908:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763902.203926:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507864576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763902.203940:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763902.203945:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763902.203947:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763902.203953:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3879 00000100:00080000:3.0:1789763902.203958:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763902.203968:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffff8d25d1b0b800 x1876699507864192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763902.203968:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763902.203983:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763902.204008:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507864704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763902.204017:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507864192/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763902.204029:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763902.204046:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763902.204051:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763902.204053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763902.204061:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3879 00000100:00080000:0.0:1789763902.204067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763902.204071:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763902.204080:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507864192:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763902.204085:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763902.204097:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763902.204113:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507864832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763902.204123:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507864320:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763902.204175:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763902.204193:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffff8d25d1b0ad80 x1876699507864320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763902.204210:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507864320/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763902.204221:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763902.204225:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763902.204229:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507864320:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763902.204236:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507864448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763902.204266:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763902.204277:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffff8d25d1b09180 x1876699507864448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763902.204292:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507864448/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763902.204310:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763902.204324:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763902.204328:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507864448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763902.204335:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507864576:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763902.204352:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763902.204364:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffff8d25d1b0aa00 x1876699507864576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763902.204408:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507864576/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763902.204420:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763902.204424:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763902.204428:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507864576:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763902.204435:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507864704:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763902.204459:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763902.204498:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffff8d25d1b09c00 x1876699507864704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763902.204514:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507864704/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763902.204524:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763902.204527:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763902.204532:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507864704:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763902.204558:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507864832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763902.204581:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763902.204594:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763902/real 1789763902] req@ffff8d25d1b08a80 x1876699507864832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763902.204608:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507864832/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763927 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763902.204618:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763902.204621:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763902.204625:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507864832:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763902.267692:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763903.291655:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763904.315662:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763905.339608:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763906.363585:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763907.323563:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763907.323581:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763907.323589:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.323603:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3885 00000100:00080000:2.0:1789763907.323615:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.323624:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763907.323670:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507864960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763907.323709:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763907.323714:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763907.323717:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.323722:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3885 00000100:00080000:2.0:1789763907.323726:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.323731:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763907.323744:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763907.323761:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507865088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763907.323778:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763907.323780:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507864960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763907.323795:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763907.323803:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.323813:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3885 00000100:00080000:2.0:1789763907.323837:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.323847:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763907.323858:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763907.323872:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763907.323888:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507865216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763907.323907:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763907.323911:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763907.323914:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.323921:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3885 00000100:00080000:2.0:1789763907.323927:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.323938:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763907.323939:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffff8d25c8022680 x1876699507864960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:2.0:1789763907.323952:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763907.323976:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507864960/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763907.323988:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507865344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763907.323998:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763907.324017:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763907.324025:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:2.0:1789763907.324032:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789763907.324039:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507864960:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763907.324042:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.324058:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3885 00000100:00080000:2.0:1789763907.324066:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763907.324072:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763907.324080:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763907.324082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763907.324110:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507865472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763907.324127:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763907.324129:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763907.324137:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763907.324142:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763907.324151:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3885 00000100:00080000:2.0:1789763907.324158:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789763907.324165:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffff8d25c8023b80 x1876699507865088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763907.324167:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763907.324182:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763907.324206:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507865088/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763907.324211:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507865600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763907.324238:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763907.324245:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763907.324254:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865088:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763907.324265:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763907.324289:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763907.324304:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffff8d25c8022300 x1876699507865216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763907.324318:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507865216/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763907.324329:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763907.324332:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763907.324336:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763907.324343:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865344:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763907.324362:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763907.324373:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffff8d25c8020380 x1876699507865344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763907.324465:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507865344/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763907.324476:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763907.324480:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763907.324484:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865344:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763907.324514:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507865472:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763907.324540:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763907.324555:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffff8d25c8022d80 x1876699507865472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763907.324570:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507865472/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763907.324580:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763907.324584:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763907.324588:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507865472:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763907.324594:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507865600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763907.324612:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763907.324624:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763907/real 1789763907] req@ffff8d25c8023100 x1876699507865600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763907.324638:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507865600/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763932 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763907.324655:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763907.324659:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763907.324663:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507865600:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763907.387642:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763908.411675:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763909.435626:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763910.459652:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763911.483690:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763912.443585:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763912.443597:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763912.443603:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763912.443615:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3890 00000100:00080000:0.0:1789763912.443627:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763912.443636:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763912.443682:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507865728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763912.443725:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763912.443731:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763912.443733:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763912.443746:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3890 00000100:00080000:0.0:1789763912.443751:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763912.443756:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763912.443768:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763912.443785:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507865856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763912.443799:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763912.443805:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763912.443808:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763912.443817:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3890 00000100:00100000:3.0:1789763912.443819:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507865728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763912.443822:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763912.443837:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763912.443845:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763912.443863:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507865984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763912.443877:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763912.443882:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763912.443885:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763912.443889:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3890 00000100:00080000:0.0:1789763912.443894:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763912.443899:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763912.443906:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763912.443921:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763912.443929:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507866112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763912.443948:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763912.443955:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763912.443959:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763912.443968:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3890 00000100:00080000:0.0:1789763912.443978:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763912.443986:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763912/real 1789763912] req@ffff8d25d25bad80 x1876699507865728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763912.443995:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763912.444017:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763912.444029:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507865728/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763912.444047:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763912.444051:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507866240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763912.444058:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763912.444074:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507865728:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763912.444097:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763912.444098:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865856:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763912.444114:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763912.444120:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763912.444128:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3890 00000100:00080000:0.0:1789763912.444137:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763912.444145:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763912.444153:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:0.0:1789763912.444161:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763912.444180:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763912/real 1789763912] req@ffff8d25d25ba300 x1876699507865856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763912.444189:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507866368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763912.444251:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507865856/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763912.444274:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763912.444282:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763912.444289:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865856:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763912.444327:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763912.444359:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763912.444448:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763912/real 1789763912] req@ffff8d25d25b8000 x1876699507865984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763912.444480:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507865984/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763912.444491:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763912.444495:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763912.444500:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507865984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763912.444508:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866112:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763912.444539:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763912.444555:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763912/real 1789763912] req@ffff8d25d25b8a80 x1876699507866112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763912.444569:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507866112/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763912.444580:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763912.444584:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763912.444587:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866112:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763912.444594:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507866240:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763912.444613:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763912.444625:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763912/real 1789763912] req@ffff8d25d25b9180 x1876699507866240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763912.444638:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507866240/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763912.444649:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763912.444653:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763912.444657:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507866240:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763912.444663:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507866368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763912.444681:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763912.444692:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763912/real 1789763912] req@ffff8d25d25bbb80 x1876699507866368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763912.444705:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507866368/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763937 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763912.444714:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763912.444718:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763912.444722:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507866368:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763912.507595:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763913.531593:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763914.555590:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763915.579581:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763916.603610:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763917.563559:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763917.563572:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763917.563578:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763917.563591:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3895 00000100:00080000:2.0:1789763917.563603:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763917.563616:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763917.563664:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507866496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763917.563717:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763917.563723:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763917.563726:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763917.563732:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3895 00000100:00080000:2.0:1789763917.563737:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763917.563744:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763917.563757:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763917.563783:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507866624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763917.563784:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507866496:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763917.563817:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763917.563822:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763917.563825:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763917.563834:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3895 00000100:00080000:2.0:1789763917.563840:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763917.563846:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763917.563856:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763917.563875:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507866752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763917.563891:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789763917.563893:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763917.563901:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763917.563907:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763917.563915:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3895 00000100:00080000:2.0:1789763917.563925:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763917.563931:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763917.563941:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763917.563966:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763917/real 1789763917] req@ffff8d25c8020e00 x1876699507866496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763917.563973:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507866880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763917.564030:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763917.564038:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:0.0:1789763917.564047:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507866496/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763917.564048:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763917.564065:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3895 00000100:00080000:2.0:1789763917.564079:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763917.564087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763917.564088:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789763917.564108:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000008:00100000:2.0:1789763917.564109:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763917.564120:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507866496:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763917.564139:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507867008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763917.564163:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866624:192.168.203.141@tcp:38:kworker.0 00000100:00080000:2.0:1789763917.564176:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763917.564183:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763917.564187:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763917.564195:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3895 00000100:00080000:2.0:1789763917.564202:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763917.564207:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763917.564215:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:0.0:1789763917.564217:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763917.564242:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507867136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763917.564254:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763917/real 1789763917] req@ffff8d25c8020700 x1876699507866624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763917.564330:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507866624/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763917.564344:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763917.564349:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763917.564435:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866624:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763917.564448:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763917.564496:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763917.564537:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763917/real 1789763917] req@ffff8d25c8021880 x1876699507866752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763917.564569:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507866752/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763917.564587:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763917.564595:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763917.564604:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763917.564620:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866880:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763917.564662:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763917.564682:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763917/real 1789763917] req@ffff8d25c8021c00 x1876699507866880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763917.564698:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507866880/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763917.564709:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763917.564713:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763917.564717:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507866880:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763917.564738:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867008:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763917.564762:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763917.564776:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763917/real 1789763917] req@ffff8d25c8023480 x1876699507867008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763917.564790:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507867008/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763917.564800:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763917.564803:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763917.564807:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867008:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763917.564813:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763917.564832:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763917.564844:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763917/real 1789763917] req@ffff8d25c8021500 x1876699507867136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763917.564858:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507867136/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763942 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763917.564876:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763917.564880:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763917.564884:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867136:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763917.627558:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763918.651529:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763919.675610:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763920.699593:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763921.723564:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763922.683591:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763922.683603:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763922.683609:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763922.683622:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3900 00000100:00080000:0.0:1789763922.683634:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763922.683644:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763922.683692:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507867264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763922.683736:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763922.683741:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763922.683744:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763922.683749:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3900 00000100:00080000:0.0:1789763922.683753:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763922.683758:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763922.683771:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763922.683789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507867392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763922.683804:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763922.683810:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763922.683814:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763922.683819:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3900 00000100:00080000:0.0:1789763922.683826:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763922.683842:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763922.683847:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507867264:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789763922.683850:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763922.683882:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507867520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763922.683899:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763922.683905:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763922.683908:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763922.683914:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3900 00000100:00080000:0.0:1789763922.683918:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763922.683922:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763922.683928:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763922.683944:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507867648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763922.683957:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763922.683961:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763922.683964:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763922.683969:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3900 00000100:00080000:0.0:1789763922.683976:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763922.683982:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763922.683991:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789763922.683997:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763922.684015:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507867776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763922.684031:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763922.684035:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763922.684039:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763922.684046:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3900 00000100:00080000:0.0:1789763922.684050:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763922.684054:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763922.684064:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763922.684087:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507867904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763922.684092:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763922/real 1789763922] req@ffff8d25d25bb480 x1876699507867264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763922.684175:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507867264/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763922.684195:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763922.684202:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763922.684208:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507867264:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763922.684220:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507867392:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763922.684251:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763922.684268:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763922/real 1789763922] req@ffff8d25d25b9f80 x1876699507867392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763922.684283:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507867392/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763922.684294:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763922.684298:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763922.684303:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507867392:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763922.684372:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507867520:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763922.684437:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763922.684454:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763922/real 1789763922] req@ffff8d25d25bb800 x1876699507867520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763922.684469:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507867520/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763922.684480:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763922.684484:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763922.684488:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507867520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763922.684496:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507867648:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763922.684518:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763922.684531:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763922/real 1789763922] req@ffff8d25d25b9c00 x1876699507867648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763922.684545:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507867648/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763922.684556:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763922.684559:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763922.684563:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507867648:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763922.684570:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867776:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763922.684590:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763922.684602:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763922/real 1789763922] req@ffff8d25d25ba680 x1876699507867776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763922.684617:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507867776/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763922.684627:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763922.684630:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763922.684634:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867776:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763922.684640:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763922.684660:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763922.684671:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763922/real 1789763922] req@ffff8d25d25bb100 x1876699507867904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763922.684685:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507867904/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763947 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763922.684696:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763922.684700:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763922.684704:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507867904:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763922.747663:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763923.771679:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763924.795610:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763925.819591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763926.843659:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763927.803513:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763927.803526:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763927.803533:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763927.803546:0:36:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3905 00000100:00080000:0.0:1789763927.803558:0:36:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763927.803567:0:36:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763927.803615:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507868032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763927.803653:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763927.803658:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763927.803661:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763927.803667:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3905 00000100:00080000:0.0:1789763927.803671:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763927.803677:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763927.803690:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763927.803718:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507868032:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763927.803719:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507868160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763927.803745:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763927.803750:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763927.803752:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763927.803758:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3905 00000100:00080000:0.0:1789763927.803762:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763927.803767:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763927.803774:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763927.803789:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507868288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763927.803802:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763927.803807:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763927.803810:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763927.803816:0:36:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3905 00000100:00080000:0.0:1789763927.803820:0:36:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763927.803823:0:36:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763927.803830:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763927.803849:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507868416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763927.803869:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763927.803879:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868160:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763927.803880:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763927.803890:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763927.803904:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3905 00000100:00080000:0.0:1789763927.803917:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763927.803928:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763927.803933:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868288:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789763927.803945:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763927.803973:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763927.803983:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507868544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763927.804008:0:36:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763927.804013:0:36:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:3.0:1789763927.804018:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507868544:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763927.804020:0:36:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763927.804037:0:36:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3905 00000100:00080000:0.0:1789763927.804045:0:36:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763927.804052:0:36:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763927.804062:0:36:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763927.804084:0:36:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507868672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763927.804147:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507868672:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763927.867578:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763928.891691:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763929.915769:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763930.875806:0:2404:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789763930.875897:0:2404:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763930.875931:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763930.876012:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763930.876025:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763930.876037:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763930.876050:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763930.876063:0:2404:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:3.0:1789763930.876109:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763927/real 1789763930] req@ffff8d25d25b9c00 x1876699507868672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763930.876166:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507868672/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763930.876205:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763930.876217:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763930.876228:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507868672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763930.876247:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763927/real 1789763930] req@ffff8d25d25ba680 x1876699507868544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763930.876322:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507868544/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763930.876341:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763930.876349:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763930.876356:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507868544:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763930.876371:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763927/real 1789763930] req@ffff8d25d25b8380 x1876699507868160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763930.876441:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507868160/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763930.876460:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763930.876469:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763930.876476:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868160:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763930.876491:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763927/real 1789763930] req@ffff8d25d25b8700 x1876699507868288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763930.876518:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507868288/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763930.876535:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763930.876542:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763930.876550:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868288:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763930.876563:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763927/real 1789763930] req@ffff8d25d25bb100 x1876699507868416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763930.876591:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507868416/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763930.876610:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763930.876620:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763930.876630:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868416:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763930.876647:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763927/real 1789763930] req@ffff8d25d25b8e00 x1876699507868032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763930.876676:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507868032/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763952 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763930.876694:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763930.876701:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763930.876707:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507868032:192.168.203.141@tcp:250:kworker.0 00010000:00010000:0.0:1789763930.939571:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763931.963668:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763932.923492:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763932.923506:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763932.923513:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763932.923539:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3910 00000100:00080000:0.0:1789763932.923553:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763932.923565:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763932.923609:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507868800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763932.923645:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763932.923650:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763932.923653:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763932.923658:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3910 00000100:00080000:0.0:1789763932.923662:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763932.923669:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763932.923682:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763932.923704:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507868928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763932.923709:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507868800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763932.923728:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763932.923735:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763932.923737:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763932.923743:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3910 00000100:00080000:0.0:1789763932.923748:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763932.923752:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763932.923758:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763932.923775:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507869056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763932.923796:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763932.923801:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763932.923804:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763932.923821:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3910 00000800:00000100:3.0:1789763932.923831:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763932.923832:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763932.923846:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763932.923853:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763932.923871:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507869184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763932.923896:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763932.923903:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763932.923907:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789763932.923919:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763932/real 1789763932] req@ffff8d25d25bb800 x1876699507868800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763932.923921:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3910 00000100:00080000:0.0:1789763932.923936:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763932.923953:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763932.923966:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763932.923977:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507868800/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763932.923998:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507869312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763932.924008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763932.924027:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763932.924031:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763932.924041:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507868800:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763932.924045:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763932.924054:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763932.924067:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3910 00000100:00080000:0.0:1789763932.924078:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763932.924086:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763932.924097:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763932.924099:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763932.924136:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507869440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763932.924170:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763932.924216:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763932/real 1789763932] req@ffff8d25d25b9f80 x1876699507868928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763932.924280:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507868928/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763932.924302:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763932.924310:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763932.924318:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507868928:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763932.924332:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763932.924421:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763932.924448:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763932/real 1789763932] req@ffff8d25d25bb480 x1876699507869056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763932.924475:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507869056/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763932.924493:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763932.924500:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763932.924507:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763932.924520:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869184:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763932.924564:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763932.924589:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763932/real 1789763932] req@ffff8d25d25bbb80 x1876699507869184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763932.924613:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507869184/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763932.924630:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763932.924637:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763932.924644:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869184:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763932.924656:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507869312:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763932.924698:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763932.924721:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763932/real 1789763932] req@ffff8d25d25b9180 x1876699507869312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763932.924743:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507869312/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763932.924758:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763932.924764:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763932.924770:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507869312:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763932.925102:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507869440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763932.925152:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763932.925176:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763932/real 1789763932] req@ffff8d25d25b8a80 x1876699507869440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763932.925200:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507869440/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763957 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763932.925230:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763932.925236:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763932.925268:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507869440:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763932.987604:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763934.011558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763935.035594:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763936.059536:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763937.083580:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763938.043510:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763938.043524:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763938.043530:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763938.043544:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3915 00000100:00080000:0.0:1789763938.043556:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763938.043565:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763938.043609:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507869568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763938.043655:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763938.043661:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763938.043663:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763938.043668:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3915 00000100:00080000:0.0:1789763938.043673:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763938.043679:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763938.043692:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763938.043716:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507869696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763938.043719:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507869568:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763938.043760:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763938.043765:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763938.043769:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763938.043776:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3915 00000100:00080000:0.0:1789763938.043789:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763938.043795:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763938.043801:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763938.043819:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507869824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789763938.043830:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763938.043847:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763938.043852:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763938.043856:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763938.043865:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3915 00000100:00080000:0.0:1789763938.043873:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763938.043880:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763938.043903:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763938.043919:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffff8d25d25b8000 x1876699507869568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763938.043934:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507869952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763938.043971:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763938.043977:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507869568/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763938.043983:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763938.043991:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763938.044010:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3915 00000100:00080000:3.0:1789763938.044011:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763938.044026:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763938.044028:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763938.044044:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763938.044046:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507869568:192.168.203.141@tcp:250:kworker.0 00000008:00100000:0.0:1789763938.044061:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763938.044085:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507870080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763938.044090:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869696:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789763938.044122:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763938.044129:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763938.044135:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789763938.044140:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763938.044154:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3915 00000100:00080000:0.0:1789763938.044166:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763938.044168:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffff8d25d25ba300 x1876699507869696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763938.044182:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763938.044199:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507869696/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789763938.044227:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763938.044256:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789763938.044269:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507870208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763938.044273:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763938.044291:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869696:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763938.044319:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869824:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763938.044358:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763938.044454:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffff8d25d25bad80 x1876699507869824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763938.044474:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507869824/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763938.044486:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763938.044491:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763938.044495:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763938.044503:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869952:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763938.044529:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763938.044542:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffff8d25d25b8a80 x1876699507869952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763938.044557:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507869952/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763938.044567:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763938.044571:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763938.044575:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507869952:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763938.044595:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870080:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763938.044616:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763938.044628:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffff8d25d25b9180 x1876699507870080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763938.044643:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507870080/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763938.044653:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763938.044656:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763938.044659:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870080:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763938.044666:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763938.044685:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763938.044697:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763938/real 1789763938] req@ffff8d25d25bbb80 x1876699507870208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763938.044711:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507870208/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763963 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763938.044720:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763938.044723:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763938.044727:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870208:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763938.107570:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763939.131614:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763940.155667:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763941.179628:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763942.203603:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763943.163596:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763943.163613:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763943.163621:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763943.163638:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3920 00000100:00080000:2.0:1789763943.163659:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763943.163675:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763943.163759:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507870336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763943.163818:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763943.163824:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763943.163828:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763943.163835:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3920 00000100:00080000:2.0:1789763943.163843:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763943.163850:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763943.163868:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763943.163891:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507870336:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763943.163893:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507870464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763943.163935:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763943.163939:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763943.163942:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763943.163948:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3920 00000100:00080000:2.0:1789763943.163953:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763943.163960:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763943.163965:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763943.163984:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507870592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763943.163997:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763943.164002:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763943.164005:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763943.164010:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3920 00000100:00080000:2.0:1789763943.164016:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763943.164020:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763943.164026:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:3.0:1789763943.164040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763943.164056:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507870720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763943.164072:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763943.164079:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763943.164082:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763943.164088:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3920 00000100:00080000:2.0:1789763943.164091:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763943.164095:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763943.164102:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763943.164115:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507870848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763943.164133:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffff8d25c8021500 x1876699507870336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763943.164140:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763943.164161:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763943.164198:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763943.164210:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3920 00000100:00080000:2.0:1789763943.164218:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763943.164224:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507870336/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763943.164232:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763943.164252:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763943.164288:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507870976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763943.164291:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.164311:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763943.164334:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507870336:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763943.164422:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507870464:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763943.164461:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763943.164481:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffff8d25c8023480 x1876699507870464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.164498:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507870464/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.164509:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.164514:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763943.164518:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507870464:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763943.164526:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507870592:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763943.164548:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763943.164561:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffff8d25c8021c00 x1876699507870592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.164575:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507870592/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.164585:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.164590:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763943.164594:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507870592:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763943.164602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507870720:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763943.164622:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763943.164634:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffff8d25c8021880 x1876699507870720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.164649:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507870720/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.164659:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.164663:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763943.164666:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507870720:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763943.164674:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870848:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763943.164693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763943.164706:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffff8d25c8020700 x1876699507870848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.164722:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507870848/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.164738:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.164745:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763943.164751:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870848:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763943.164763:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870976:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763943.164803:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763943.164827:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763943/real 1789763943] req@ffff8d25c8020e00 x1876699507870976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763943.164847:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507870976/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763968 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763943.164857:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763943.164862:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763943.164866:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507870976:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763943.227562:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763944.251616:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763945.275637:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763946.299563:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763947.323536:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763948.283444:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.283452:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763948.283455:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.283465:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3925 00000100:00080000:2.0:1789763948.283473:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.283479:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763948.283523:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507871104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763948.283550:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.283554:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763948.283556:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.283562:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3925 00000100:00080000:2.0:1789763948.283566:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.283571:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763948.283581:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763948.283582:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507871104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763948.283596:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507871232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763948.283605:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.283608:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763948.283609:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.283612:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3925 00000100:00080000:2.0:1789763948.283614:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.283617:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763948.283621:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763948.283628:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507871360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763948.283636:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.283638:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763948.283639:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.283641:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3925 00000100:00080000:2.0:1789763948.283643:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.283648:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763948.283649:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763948.283652:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763948.283661:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507871488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763948.283669:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763948.283671:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763948.283672:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.283675:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3925 00000100:00080000:2.0:1789763948.283677:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.283679:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763948.283682:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763948.283688:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507871616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763948.283695:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789763948.283696:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763948] req@ffff8d25c8023100 x1876699507871104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763948.283699:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763948.283703:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763948.283711:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3925 00000100:00080000:2.0:1789763948.283715:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763948.283718:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763948.283722:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763948.283724:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507871104/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763948.283732:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507871744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763948.283738:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763948.283743:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763948.283748:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507871104:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763948.283767:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507871232:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763948.283786:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763948.283794:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763948] req@ffff8d25c8022d80 x1876699507871232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763948.283803:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507871232/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763948.283817:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763948.283821:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763948.283824:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507871232:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763948.283830:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507871360:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763948.283852:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763948.283863:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763948] req@ffff8d25c8020380 x1876699507871360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763948.283872:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507871360/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763948.283878:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763948.283880:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763948.283882:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507871360:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763948.283886:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507871488:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763948.283900:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763948.283907:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763948] req@ffff8d25c8022300 x1876699507871488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763948.283915:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507871488/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763948.283920:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763948.283921:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763948.283924:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507871488:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763948.283927:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507871616:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763948.283938:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763948.283944:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763948] req@ffff8d25c8023b80 x1876699507871616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763948.283951:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507871616/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763948.283957:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763948.283958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763948.283961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507871616:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763948.283964:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507871744:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763948.283974:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763948.283980:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763948/real 1789763948] req@ffff8d25c8022680 x1876699507871744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763948.283987:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507871744/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763973 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763948.283992:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763948.283994:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763948.283995:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507871744:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763948.347675:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763949.371556:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763950.395579:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763951.419618:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763952.443637:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763953.403483:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763953.403493:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763953.403499:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763953.403510:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3931 00000100:00080000:3.0:1789763953.403529:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763953.403538:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763953.403572:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507871872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763953.403604:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763953.403609:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763953.403611:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763953.403616:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3931 00000100:00080000:3.0:1789763953.403620:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763953.403625:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763953.403633:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763953.403637:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507871872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763953.403649:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507872000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763953.403662:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763953.403666:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763953.403668:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763953.403673:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3931 00000100:00080000:3.0:1789763953.403688:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763953.403694:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763953.403700:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763953.403702:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763953.403717:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507872128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763953.403730:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763953.403733:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763953.403734:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763953.403737:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3931 00000100:00080000:3.0:1789763953.403741:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763953.403742:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763953/real 1789763953] req@ffff8d25d1b08700 x1876699507871872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763953.403746:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763953.403755:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763953.403766:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507871872/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763953.403767:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507872256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763953.403808:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763953.403813:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763953.403816:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507871872:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763953.403836:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872000:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763953.403871:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763953.403882:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763953.403887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763953/real 1789763953] req@ffff8d25d1b08e00 x1876699507872000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763953.403888:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763953.403891:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763953.403899:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3931 00000100:00080000:3.0:1789763953.403905:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763953.403911:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763953.403915:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763953.403920:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507872000/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763953.403939:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507872384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763953.403953:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763953.403956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763953.403959:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872000:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763953.403965:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872128:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763953.403984:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763953.404003:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763953/real 1789763953] req@ffff8d25d1b09880 x1876699507872128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763953.404014:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507872128/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763953.404020:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763953.404022:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763953.404024:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872128:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763953.404037:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872256:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763953.404053:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763953.404064:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763953.404066:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763953/real 1789763953] req@ffff8d25d1b0bb80 x1876699507872256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763953.404069:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763953.404073:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763953.404080:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507872256/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763953.404087:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763953.404103:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763953.404110:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872256:192.168.203.141@tcp:8:kworker.0 00000100:00080000:3.0:1789763953.404122:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3931 00000100:00100000:2.0:1789763953.404126:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507872384:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789763953.404129:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763953.404145:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763953.404151:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:2.0:1789763953.404157:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763953.404166:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507872512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763953.404176:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763953/real 1789763953] req@ffff8d25d1b0a680 x1876699507872384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763953.404189:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507872384/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763953.404195:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763953.404198:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763953.404201:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507872384:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763953.404211:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507872512:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763953.404222:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763953.404229:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763953/real 1789763953] req@ffff8d25d1b0bb80 x1876699507872512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763953.404236:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507872512/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763978 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763953.404241:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763953.404243:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763953.404244:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507872512:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763953.467539:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763954.491647:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763955.515536:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763956.539519:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763957.563565:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763958.523525:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763958.523550:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763958.523557:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763958.523572:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3936 00000100:00080000:3.0:1789763958.523585:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763958.523595:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763958.523646:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507872640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763958.523680:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763958.523685:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763958.523688:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763958.523693:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3936 00000100:00080000:3.0:1789763958.523697:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763958.523702:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763958.523715:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763958.523730:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507872768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763958.523745:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763958.523751:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763958.523754:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763958.523765:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3936 00000100:00100000:2.0:1789763958.523766:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507872640:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763958.523769:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763958.523782:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763958.523791:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763958.523809:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507872896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763958.523821:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763958.523826:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763958.523828:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763958.523833:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3936 00000100:00080000:3.0:1789763958.523849:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763958.523855:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763958.523860:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763958.523873:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507873024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763958.523886:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:2.0:1789763958.523896:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763958.523896:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763958.523904:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763958.523913:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3936 00000100:00080000:3.0:1789763958.523918:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763958.523924:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763958.523931:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763958.523946:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507873152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763958.523958:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763958.523963:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763958.523965:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763958.523970:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3936 00000100:00080000:3.0:1789763958.523975:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763958.523980:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763958.523987:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763958.524000:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763958/real 1789763958] req@ffff8d25d1b09880 x1876699507872640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763958.524008:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507873280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763958.524104:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507872640/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763958.524133:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763958.524142:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763958.524150:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507872640:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763958.524193:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872768:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763958.524243:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763958.524272:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763958/real 1789763958] req@ffff8d25d1b08e00 x1876699507872768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763958.524299:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507872768/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763958.524317:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763958.524324:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763958.524331:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872768:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763958.524343:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872896:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763958.524435:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763958.524464:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763958/real 1789763958] req@ffff8d25d1b08700 x1876699507872896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763958.524488:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507872896/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763958.524504:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763958.524511:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763958.524517:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507872896:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763958.524530:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873024:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763958.524571:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763958.524595:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763958/real 1789763958] req@ffff8d25d1b08a80 x1876699507873024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763958.524619:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507873024/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763958.524635:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763958.524642:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763958.524649:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873024:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763958.524661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507873152:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763958.524700:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763958.524723:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763958/real 1789763958] req@ffff8d25d1b09c00 x1876699507873152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763958.524747:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507873152/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763958.524763:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763958.524769:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763958.524776:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507873152:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763958.524788:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507873280:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763958.524825:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763958.524846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763958/real 1789763958] req@ffff8d25d1b0aa00 x1876699507873280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763958.524870:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507873280/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763983 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763958.524887:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763958.524893:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763958.524900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507873280:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763958.587649:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763959.611591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763960.635648:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763961.659655:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763962.683683:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763963.643530:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763963.643545:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763963.643551:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763963.643564:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3941 00000100:00080000:3.0:1789763963.643577:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763963.643588:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763963.643644:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507873408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763963.643692:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763963.643701:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763963.643705:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763963.643712:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3941 00000100:00080000:3.0:1789763963.643719:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763963.643726:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763963.643741:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763963.643756:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507873408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763963.643767:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507873536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763963.643784:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763963.643789:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763963.643792:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763963.643797:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3941 00000100:00080000:3.0:1789763963.643801:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763963.643806:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763963.643813:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763963.643829:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507873664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763963.643842:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763963.643847:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763963.643849:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763963.643853:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3941 00000100:00080000:3.0:1789763963.643858:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763963.643867:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:2.0:1789763963.643870:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789763963.643875:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763963.643892:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507873792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763963.643907:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763963.643912:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763963.643914:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763963.643919:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3941 00000100:00080000:3.0:1789763963.643924:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763963.643928:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763963.643936:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763963.643946:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763963/real 1789763963] req@ffff8d25d1b09180 x1876699507873408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763963.643955:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507873920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763963.644005:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763963.644011:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763963.644014:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:2.0:1789763963.644053:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507873408/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763963.644058:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3941 00000100:00080000:3.0:1789763963.644068:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763963.644082:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763963.644096:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:2.0:1789763963.644100:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763963.644112:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763963.644124:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507873408:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763963.644126:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507874048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763963.644169:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873536:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763963.644215:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763963.644235:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763963/real 1789763963] req@ffff8d25d1b0ad80 x1876699507873536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763963.644253:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507873536/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763963.644264:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763963.644268:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763963.644272:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873536:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763963.644279:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873664:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763963.644300:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763963.644313:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763963/real 1789763963] req@ffff8d25d1b0b800 x1876699507873664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763963.644328:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507873664/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763963.644338:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763963.644341:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763963.644345:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873664:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763963.644352:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873792:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763963.644372:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763963.644421:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763963/real 1789763963] req@ffff8d25d1b0b480 x1876699507873792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763963.644437:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507873792/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763963.644447:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763963.644451:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763963.644455:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507873792:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763963.644461:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507873920:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763963.644484:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763963.644498:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763963/real 1789763963] req@ffff8d25d1b08000 x1876699507873920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763963.644522:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507873920/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763963.644537:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763963.644544:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763963.644550:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507873920:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763963.644563:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507874048:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763963.644603:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763963.644624:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763963/real 1789763963] req@ffff8d25d1b0a300 x1876699507874048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763963.644650:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507874048/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763988 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763963.644660:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763963.644664:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763963.644668:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507874048:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763963.707591:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763964.731572:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763965.755611:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763966.779654:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763967.803624:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789763968.763541:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763968.763550:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789763968.763554:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763968.763562:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3946 00000100:00080000:0.0:1789763968.763570:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763968.763578:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789763968.763616:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507874176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763968.763645:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763968.763648:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763968.763650:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763968.763653:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3946 00000100:00080000:0.0:1789763968.763656:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763968.763659:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763968.763667:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763968.763678:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507874304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763968.763687:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763968.763688:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507874176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763968.763693:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763968.763697:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763968.763700:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3946 00000100:00080000:0.0:1789763968.763703:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763968.763706:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763968.763711:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763968.763733:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507874432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763968.763750:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763968.763754:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789763968.763756:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:2.0:1789763968.763762:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789763968.763763:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3946 00000100:00080000:0.0:1789763968.763772:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789763968.763776:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763968.763781:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789763968.763793:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507874560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763968.763801:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763968.763805:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789763968.763807:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789763968.763810:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3946 00000100:00080000:0.0:1789763968.763813:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763968.763813:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763968/real 1789763968] req@ffff8d25d25bb480 x1876699507874176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763968.763819:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763968.763825:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763968.763837:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507874688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763968.763840:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507874176/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763968.763852:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789763968.763857:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789763968.763860:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763968.763860:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763968.763870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789763968.763871:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3946 00000100:00080000:0.0:1789763968.763876:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763968.763877:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507874176:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789763968.763882:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789763968.763887:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763968.763894:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507874816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763968.763906:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507874304:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763968.763928:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763968.763951:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763968/real 1789763968] req@ffff8d25d25b9f80 x1876699507874304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763968.763966:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507874304/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763968.763996:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763968.764000:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763968.764003:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507874304:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763968.764021:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507874432:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763968.764041:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763968.764051:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763968/real 1789763968] req@ffff8d25d25bb800 x1876699507874432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763968.764059:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507874432/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763968.764065:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763968.764067:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763968.764069:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507874432:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763968.764073:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507874560:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763968.764085:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763968.764091:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763968/real 1789763968] req@ffff8d25d25b8e00 x1876699507874560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763968.764098:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507874560/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763968.764103:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763968.764105:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763968.764107:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507874560:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763968.764111:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507874688:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763968.764121:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763968.764127:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763968/real 1789763968] req@ffff8d25d25bb100 x1876699507874688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763968.764134:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507874688/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763968.764139:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763968.764141:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763968.764142:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507874688:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763968.764146:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507874816:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763968.764155:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763968.764160:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763968/real 1789763968] req@ffff8d25d25b8700 x1876699507874816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763968.764167:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507874816/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763993 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763968.764172:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763968.764174:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763968.764176:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507874816:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763968.827597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763969.851619:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763970.875625:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763971.899664:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763972.923592:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763973.883624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763973.883635:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763973.883640:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763973.883663:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3951 00000100:00080000:3.0:1789763973.883680:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763973.883688:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763973.883759:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507874944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763973.883817:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763973.883820:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763973.883823:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763973.883827:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3951 00000100:00080000:3.0:1789763973.883831:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763973.883835:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763973.883846:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763973.883879:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507875072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763973.883896:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:2.0:1789763973.883898:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507874944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763973.883902:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763973.883908:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763973.883921:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3951 00000100:00080000:3.0:1789763973.883925:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763973.883930:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763973.883936:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763973.883994:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507875200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763973.884010:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763973.884016:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763973.884019:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763973.884026:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3951 00000100:00080000:3.0:1789763973.884032:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000800:00000100:2.0:1789763973.884039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763973.884045:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763973.884067:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763973.884089:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507875328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763973.884105:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763973.884113:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00100000:2.0:1789763973.884115:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763973/real 1789763973] req@ffff8d25d1b09500 x1876699507874944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763973.884117:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763973.884131:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3951 00000100:00080000:3.0:1789763973.884144:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763973.884152:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763973.884159:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763973.884165:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507874944/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763973.884177:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507875456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763973.884200:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763973.884201:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763973.884205:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763973.884211:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763973.884216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789763973.884221:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3951 00000100:00080000:3.0:1789763973.884230:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:2.0:1789763973.884252:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507874944:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763973.884254:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763973.884264:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763973.884281:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507875584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763973.884309:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875072:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763973.884357:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763973.884376:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763973/real 1789763973] req@ffff8d25d1b08380 x1876699507875072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763973.884445:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507875072/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763973.884461:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763973.884468:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763973.884475:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875072:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763973.884497:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875200:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763973.884536:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763973.884576:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763973/real 1789763973] req@ffff8d25d1b0b100 x1876699507875200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763973.884606:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507875200/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763973.884618:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763973.884623:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763973.884628:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875200:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763973.884638:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875328:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763973.884675:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763973.884688:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763973/real 1789763973] req@ffff8d25d1b0a300 x1876699507875328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763973.884699:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507875328/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763973.884707:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763973.884710:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763973.884713:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875328:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763973.884718:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507875456:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763973.884732:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763973.884741:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763973/real 1789763973] req@ffff8d25d1b08000 x1876699507875456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763973.884752:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507875456/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763973.884765:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763973.884768:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763973.884771:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507875456:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763973.884776:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507875584:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763973.884820:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763973.884831:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763973/real 1789763973] req@ffff8d25d1b0b480 x1876699507875584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763973.884843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507875584/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789763998 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763973.884852:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763973.884855:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763973.884859:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507875584:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763973.947621:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763974.971591:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763975.995530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763977.019622:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763978.043567:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763979.003481:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.003489:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763979.003493:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.003502:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3956 00000100:00080000:3.0:1789763979.003512:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.003518:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763979.003558:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507875712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763979.003585:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.003589:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763979.003591:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.003596:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3956 00000100:00080000:3.0:1789763979.003600:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.003606:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763979.003616:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763979.003630:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507875840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763979.003631:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507875712:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763979.003650:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.003653:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763979.003654:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.003657:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3956 00000100:00080000:3.0:1789763979.003660:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.003663:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763979.003666:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763979.003675:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507875968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763979.003681:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.003684:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763979.003685:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.003687:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3956 00000100:00080000:3.0:1789763979.003690:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.003692:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763979.003695:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763979.003702:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507876096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:2.0:1789763979.003709:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763979.003714:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.003717:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763979.003718:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.003722:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3956 00000100:00080000:3.0:1789763979.003725:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.003728:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763979.003731:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763979.003738:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507876224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763979.003743:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763979.003745:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763979.003746:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763979.003749:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3956 00000100:00080000:3.0:1789763979.003751:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763979.003754:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763979.003758:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763979.003763:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffff8d25d1b0b800 x1876699507875712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763979.003769:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507876352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:2.0:1789763979.003807:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507875712/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763979.003820:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763979.003826:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763979.003831:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507875712:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789763979.003855:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875840:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763979.003882:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763979.003897:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffff8d25d1b0ad80 x1876699507875840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763979.003925:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507875840/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763979.003936:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763979.003939:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763979.003943:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875840:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763979.003950:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875968:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763979.003973:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763979.003987:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffff8d25d1b09180 x1876699507875968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763979.004000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507875968/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763979.004009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763979.004013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763979.004016:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507875968:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763979.004023:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876096:192.168.203.141@tcp:8:kworker.0 00000800:00000100:2.0:1789763979.004043:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763979.004055:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffff8d25d1b0aa00 x1876699507876096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763979.004069:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507876096/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763979.004077:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763979.004081:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763979.004084:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876096:192.168.203.141@tcp:8:kworker.0 00000100:00100000:2.0:1789763979.004091:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507876224:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763979.004109:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763979.004121:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffff8d25d1b09c00 x1876699507876224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763979.004133:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507876224/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763979.004142:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763979.004145:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763979.004148:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507876224:192.168.203.141@tcp:38:kworker.0 00000100:00100000:2.0:1789763979.004154:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507876352:192.168.203.141@tcp:38:kworker.0 00000800:00000100:2.0:1789763979.004173:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789763979.004184:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763979/real 1789763979] req@ffff8d25d1b08a80 x1876699507876352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763979.004204:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507876352/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764004 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763979.004213:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:2.0:1789763979.004216:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763979.004220:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507876352:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763979.067516:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763980.091558:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763981.115626:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763982.139613:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763983.163596:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763984.123568:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763984.123581:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763984.123588:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763984.123601:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3961 00000100:00080000:2.0:1789763984.123614:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763984.123624:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763984.123673:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507876480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763984.123714:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763984.123719:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763984.123721:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763984.123727:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3961 00000100:00080000:2.0:1789763984.123732:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763984.123737:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763984.123749:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763984.123766:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507876608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763984.123784:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507876480:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763984.123786:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763984.123801:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763984.123804:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763984.123811:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3961 00000100:00080000:2.0:1789763984.123814:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763984.123820:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763984.123827:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763984.123846:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507876736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763984.123858:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763984.123863:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763984.123887:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763984.123893:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3961 00000100:00080000:2.0:1789763984.123898:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763984.123916:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:3.0:1789763984.123927:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:2.0:1789763984.123930:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763984.123950:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507876864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763984.123966:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763984.123970:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763984.123973:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763984.123978:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3961 00000100:00080000:2.0:1789763984.123984:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763984.123989:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763984.123995:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763984.124023:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507876992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763984.124025:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffff8d25c8023800 x1876699507876480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763984.124064:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763984.124071:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763984.124074:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763984.124081:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3961 00000100:00080000:2.0:1789763984.124085:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:3.0:1789763984.124090:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507876480/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763984.124104:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763984.124118:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789763984.124122:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763984.124132:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:2.0:1789763984.124139:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507877120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789763984.124148:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507876480:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789763984.124205:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876608:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763984.124258:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763984.124280:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffff8d25c8020000 x1876699507876608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763984.124298:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507876608/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763984.124310:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763984.124315:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763984.124320:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876608:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763984.124329:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876736:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763984.124355:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763984.124369:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffff8d25c8022a00 x1876699507876736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763984.124440:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507876736/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763984.124460:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763984.124467:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763984.124474:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876736:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763984.124486:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876864:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789763984.124521:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763984.124537:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffff8d25c8021180 x1876699507876864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763984.124553:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507876864/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763984.124564:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763984.124568:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763984.124572:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507876864:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789763984.124579:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507876992:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763984.124599:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763984.124612:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffff8d25c8020a80 x1876699507876992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763984.124626:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507876992/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763984.124645:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763984.124648:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763984.124652:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507876992:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789763984.124658:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507877120:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789763984.124677:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789763984.124689:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763984/real 1789763984] req@ffff8d25c8022680 x1876699507877120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789763984.124712:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507877120/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764009 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789763984.124722:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789763984.124725:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789763984.124729:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507877120:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763984.187610:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763985.211527:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763986.235578:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763987.259574:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763988.283575:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763989.243472:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763989.243480:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763989.243484:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763989.243495:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3966 00000100:00080000:2.0:1789763989.243508:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763989.243518:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763989.243562:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507877248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763989.243594:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763989.243597:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763989.243598:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763989.243602:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3966 00000100:00080000:2.0:1789763989.243605:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763989.243608:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763989.243616:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763989.243628:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507877376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763989.243642:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763989.243651:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789763989.243652:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507877248:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763989.243659:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763989.243673:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3966 00000100:00080000:2.0:1789763989.243677:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763989.243680:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763989.243684:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763989.243697:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507877504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763989.243706:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763989.243709:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763989.243710:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763989.243714:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3966 00000100:00080000:2.0:1789763989.243721:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763989.243726:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763989.243730:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763989.243737:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507877632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789763989.243750:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789763989.243766:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763989.243769:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763989.243771:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763989.243777:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3966 00000100:00080000:2.0:1789763989.243783:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763989.243786:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763989.243789:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763989.243802:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507877760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763989.243810:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763989.243817:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffff8d25c8023b80 x1876699507877248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789763989.243817:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763989.243823:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763989.243846:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3966 00000100:00080000:2.0:1789763989.243857:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763989.243866:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763989.243873:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789763989.243880:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507877248/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789763989.243886:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507877888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789763989.243931:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763989.243942:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763989.243950:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507877248:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763989.243988:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507877376:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763989.244018:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763989.244044:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffff8d25c8022300 x1876699507877376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763989.244061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507877376/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763989.244069:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763989.244073:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763989.244077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507877376:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763989.244082:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507877504:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763989.244099:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763989.244107:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffff8d25c8020380 x1876699507877504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763989.244115:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507877504/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763989.244120:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763989.244122:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763989.244124:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507877504:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763989.244127:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507877632:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763989.244138:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763989.244144:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffff8d25c8022d80 x1876699507877632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763989.244152:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507877632/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763989.244162:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763989.244164:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763989.244166:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507877632:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763989.244170:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507877760:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763989.244180:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763989.244186:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffff8d25c8023100 x1876699507877760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763989.244193:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507877760/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763989.244199:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763989.244200:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763989.244202:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507877760:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763989.244206:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507877888:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763989.244216:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763989.244222:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763989/real 1789763989] req@ffff8d25c8020e00 x1876699507877888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763989.244228:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507877888/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764014 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763989.244233:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763989.244235:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763989.244238:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507877888:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789763989.307542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763990.331597:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789763991.355522:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763992.379530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763993.403512:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789763994.363484:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.363492:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789763994.363496:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.363505:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3972 00000100:00080000:2.0:1789763994.363513:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.363519:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789763994.363551:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020700 x1876699507878016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763994.363573:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.363576:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763994.363577:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.363580:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3972 00000100:00080000:2.0:1789763994.363583:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.363586:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763994.363595:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763994.363604:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021880 x1876699507878144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763994.363614:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789763994.363615:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507878016:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789763994.363617:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763994.363621:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.363627:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3972 00000100:00080000:2.0:1789763994.363630:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.363632:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763994.363635:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763994.363643:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021c00 x1876699507878272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763994.363649:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.363651:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789763994.363652:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.363655:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3972 00000100:00080000:2.0:1789763994.363657:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.363659:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763994.363663:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789763994.363670:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023480 x1876699507878400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763994.363677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.363680:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789763994.363681:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.363683:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3972 00000100:00080000:2.0:1789763994.363686:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.363688:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763994.363691:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763994.363697:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021500 x1876699507878528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789763994.363708:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789763994.363710:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789763994.363711:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789763994.363715:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3972 00000100:00080000:2.0:1789763994.363716:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789763994.363719:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789763994.363723:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789763994.363731:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507878656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763994.363737:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763994.363774:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763994.363829:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763994.363856:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507878528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763994.363892:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507878656:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763994.427534:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763995.451506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789763996.475549:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000400:00000100:1.0:1789763997.435660:0:2405:0:(lib-socket.c:438:lnet_sock_connect()) Error -113 connecting 1023 -> 192.168.203.141:988 00000400:00000100:1.0:1789763997.435704:0:2405:0:(acceptor.c:117:lnet_connect_console_error()) Connection to 192.168.203.141@tcp at host 192.168.203.141 was unreachable: the network or that node may be down, or Lustre may be misconfigured. 00000800:00000100:1.0:1789763997.435721:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763997.435760:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763997.435792:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763997.435805:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789763997.435806:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763997] req@ffff8d25c8021880 x1876699507878144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000800:00000100:1.0:1789763997.435817:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000800:00000100:1.0:1789763997.435832:0:2405:0:(socklnd_cb.c:353:ksocknal_txlist_done()) Deleting packet type 1 len 520 0@<0:0>->192.168.203.141@tcp 00000100:00100000:0.0:1789763997.435849:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021880 x1876699507878144/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763997.435863:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763997.435868:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763997.435873:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878144:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763997.435881:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763997] req@ffff8d25c8021c00 x1876699507878272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763997.435895:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021c00 x1876699507878272/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763997.435905:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763997.435909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763997.435913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878272:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763997.435920:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763997] req@ffff8d25c8023480 x1876699507878400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763997.435931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023480 x1876699507878400/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763997.435940:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763997.435943:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763997.435947:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878400:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763997.435953:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763997] req@ffff8d25c8021500 x1876699507878528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763997.435965:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021500 x1876699507878528/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763997.435973:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763997.435977:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763997.435980:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507878528:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763997.435987:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763997] req@ffff8d25c8020700 x1876699507878016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763997.436000:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020700 x1876699507878016/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763997.436010:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763997.436013:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763997.436017:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507878016:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763997.436047:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763994/real 1789763997] req@ffff8d25c8020e00 x1876699507878656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763997.436061:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507878656/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764019 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763997.436070:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763997.436074:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763997.436077:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507878656:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789763997.499541:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789763998.523514:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789763999.483499:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763999.483508:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789763999.483511:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763999.483520:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3977 00000100:00080000:3.0:1789763999.483530:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763999.483539:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789763999.483570:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08700 x1876699507878784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763999.483591:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763999.483594:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763999.483595:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763999.483599:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3977 00000100:00080000:3.0:1789763999.483601:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763999.483605:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763999.483614:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763999.483624:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08e00 x1876699507878912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763999.483633:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763999.483636:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763999.483639:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789763999.483645:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507878784:192.168.203.141@tcp:250:kworker.0 00000100:00080000:3.0:1789763999.483645:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3977 00000100:00080000:3.0:1789763999.483650:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763999.483657:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763999.483663:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763999.483670:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09880 x1876699507879040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763999.483677:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763999.483680:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789763999.483681:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763999.483685:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3977 00000100:00080000:3.0:1789763999.483687:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763999.483689:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763999.483692:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789763999.483698:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0bb80 x1876699507879168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763999.483705:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763999.483708:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789763999.483709:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763999.483711:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3977 00000100:00080000:3.0:1789763999.483713:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789763999.483716:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763999.483719:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763999.483729:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a680 x1876699507879296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789763999.483739:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789763999.483742:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789763999.483744:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789763999.483749:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3977 00000100:00080000:3.0:1789763999.483767:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000800:00000100:0.0:1789763999.483775:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789763999.483778:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789763999.483786:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789763999.483804:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507879424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789763999.483855:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763999/real 1789763999] req@ffff8d25d1b08700 x1876699507878784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763999.483883:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08700 x1876699507878784/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763999.483897:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763999.483902:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763999.483907:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507878784:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789763999.483947:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878912:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763999.483978:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763999.483993:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763999/real 1789763999] req@ffff8d25d1b08e00 x1876699507878912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763999.484006:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08e00 x1876699507878912/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763999.484015:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763999.484019:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763999.484022:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507878912:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763999.484029:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879040:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763999.484049:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763999.484062:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763999/real 1789763999] req@ffff8d25d1b09880 x1876699507879040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763999.484074:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09880 x1876699507879040/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763999.484094:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763999.484098:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763999.484101:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09880 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879040:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763999.484109:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879168:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789763999.484130:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763999.484142:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763999/real 1789763999] req@ffff8d25d1b0bb80 x1876699507879168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763999.484161:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0bb80 x1876699507879168/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763999.484170:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763999.484173:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763999.484176:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0bb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879168:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789763999.484182:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507879296:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763999.484204:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763999.484216:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763999/real 1789763999] req@ffff8d25d1b0a680 x1876699507879296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763999.484227:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a680 x1876699507879296/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763999.484235:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763999.484238:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763999.484241:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507879296:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789763999.484248:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507879424:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789763999.484269:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789763999.484282:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789763999/real 1789763999] req@ffff8d25d1b08a80 x1876699507879424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789763999.484294:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507879424/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764024 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789763999.484309:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789763999.484312:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789763999.484316:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507879424:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789763999.547488:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764000.571530:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764001.595539:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764002.619510:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764003.643557:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764004.603484:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764004.603499:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764004.603512:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764004.603526:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3982 00000100:00080000:2.0:1789764004.603536:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764004.603544:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789764004.603588:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507879552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764004.603614:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764004.603617:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764004.603619:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764004.603624:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3982 00000100:00080000:2.0:1789764004.603628:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764004.603633:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764004.603645:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764004.603661:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507879552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789764004.603664:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507879680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764004.603679:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764004.603682:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764004.603684:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764004.603688:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3982 00000100:00080000:2.0:1789764004.603691:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764004.603694:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764004.603699:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764004.603711:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507879808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764004.603736:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764004.603739:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764004.603740:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764004.603744:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3982 00000100:00080000:2.0:1789764004.603748:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764004.603751:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764004.603756:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789764004.603762:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789764004.603767:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507879936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764004.603777:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764004.603780:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764004.603782:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764004.603784:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3982 00000100:00080000:2.0:1789764004.603787:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764004.603790:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764004.603792:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764004.603801:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507880064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764004.603807:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764004.603810:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00100000:0.0:1789764004.603814:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764004/real 1789764004] req@ffff8d25c8023100 x1876699507879552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764004.603815:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764004.603823:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3982 00000100:00080000:2.0:1789764004.603829:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764004.603834:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764004.603838:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764004.603843:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507879552/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764004.603848:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507880192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764004.603864:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764004.603870:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764004.603873:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507879552:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789764004.603897:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879680:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764004.603914:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764004.603923:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764004/real 1789764004] req@ffff8d25c8022d80 x1876699507879680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764004.603931:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507879680/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764004.603937:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764004.603940:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764004.603943:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879680:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764004.603947:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879808:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764004.603958:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764004.603964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764004/real 1789764004] req@ffff8d25c8020380 x1876699507879808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764004.603972:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507879808/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764004.603977:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764004.603979:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764004.603981:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879808:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764004.603985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879936:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789764004.603996:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764004.604002:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764004/real 1789764004] req@ffff8d25c8022300 x1876699507879936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764004.604010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507879936/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764004.604016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764004.604017:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764004.604019:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507879936:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789764004.604022:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880064:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764004.604033:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764004.604040:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764004/real 1789764004] req@ffff8d25c8023b80 x1876699507880064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764004.604048:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507880064/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764004.604053:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764004.604054:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764004.604056:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880064:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764004.604059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880192:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764004.604070:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764004.604076:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764004/real 1789764004] req@ffff8d25c8022680 x1876699507880192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764004.604084:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507880192/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764029 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764004.604088:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764004.604090:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764004.604092:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880192:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789764004.667542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764005.691542:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764006.715564:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764007.739522:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764008.763562:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764009.723504:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764009.723515:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764009.723521:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764009.723535:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3987 00000100:00080000:2.0:1789764009.723547:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764009.723554:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789764009.723598:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020a80 x1876699507880320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764009.723625:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764009.723628:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764009.723629:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764009.723633:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3987 00000100:00080000:2.0:1789764009.723636:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764009.723639:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764009.723648:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764009.723658:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8021180 x1876699507880448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764009.723667:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764009.723671:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764009.723672:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764009.723693:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3987 00000100:00100000:0.0:1789764009.723698:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507880320:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789764009.723700:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764009.723714:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764009.723720:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764009.723736:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022a00 x1876699507880576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764009.723747:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764009.723750:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764009.723752:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764009.723756:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3987 00000100:00080000:2.0:1789764009.723759:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764009.723761:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764009.723765:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:2.0:1789764009.723773:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020000 x1876699507880704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764009.723785:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:2.0:1789764009.723785:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764009.723795:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764009.723797:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764009.723802:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3987 00000100:00080000:2.0:1789764009.723805:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764009.723810:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764009.723814:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764009.723822:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023800 x1876699507880832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764009.723830:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764009.723833:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764009.723834:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764009.723839:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3987 00000100:00080000:2.0:1789764009.723843:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764009.723846:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764009/real 1789764009] req@ffff8d25c8020a80 x1876699507880320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764009.723851:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764009.723861:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764009.723881:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022680 x1876699507880960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764009.723938:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020a80 x1876699507880320/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764009.723952:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764009.723956:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764009.723961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507880320:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789764009.723985:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507880448:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764009.724008:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764009.724020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764009/real 1789764009] req@ffff8d25c8021180 x1876699507880448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764009.724033:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8021180 x1876699507880448/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764009.724043:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764009.724046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764009.724050:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8021180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507880448:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764009.724056:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507880576:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764009.724072:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764009.724084:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764009/real 1789764009] req@ffff8d25c8022a00 x1876699507880576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764009.724097:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022a00 x1876699507880576/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764009.724113:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764009.724117:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764009.724120:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022a00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507880576:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764009.724127:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507880704:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789764009.724146:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764009.724156:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764009/real 1789764009] req@ffff8d25c8020000 x1876699507880704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764009.724169:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020000 x1876699507880704/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764009.724178:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764009.724181:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764009.724184:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507880704:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789764009.724190:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880832:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764009.724206:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764009.724216:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764009/real 1789764009] req@ffff8d25c8023800 x1876699507880832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764009.724228:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023800 x1876699507880832/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764009.724237:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764009.724240:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764009.724244:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880832:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764009.724250:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880960:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764009.724266:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764009.724275:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764009/real 1789764009] req@ffff8d25c8022680 x1876699507880960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764009.724288:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022680 x1876699507880960/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764034 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764009.724296:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764009.724299:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764009.724302:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507880960:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789764009.787542:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764010.811570:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764011.835616:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764012.859546:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764013.883555:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764014.843501:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764014.843515:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764014.843521:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764014.843534:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3992 00000100:00080000:3.0:1789764014.843547:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764014.843556:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789764014.843604:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507881088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764014.843635:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764014.843640:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764014.843657:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764014.843663:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3992 00000100:00080000:3.0:1789764014.843667:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764014.843672:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764014.843684:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764014.843700:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507881088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789764014.843701:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507881216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764014.843720:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764014.843724:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764014.843726:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764014.843730:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3992 00000100:00080000:3.0:1789764014.843734:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764014.843738:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764014.843744:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764014.843756:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507881344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764014.843774:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764014.843778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764014.843780:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764014.843785:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3992 00000100:00080000:3.0:1789764014.843789:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764014.843795:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000800:00000100:0.0:1789764014.843797:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000008:00100000:3.0:1789764014.843803:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764014.843824:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507881472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764014.843836:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764014.843840:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764014.843842:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764014.843846:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3992 00000100:00080000:3.0:1789764014.843851:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764014.843856:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789764014.843861:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764014/real 1789764014] req@ffff8d25d1b09c00 x1876699507881088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789764014.843862:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764014.843878:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507881600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764014.843889:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789764014.843891:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507881088/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764014.843894:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764014.843898:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764014.843906:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3992 00000100:00080000:3.0:1789764014.843913:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764014.843918:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764014.843918:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764014.843926:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789764014.843929:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764014.843936:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507881088:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789764014.843941:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507881728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764014.843975:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881216:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764014.844003:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764014.844018:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764014/real 1789764014] req@ffff8d25d1b0aa00 x1876699507881216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764014.844044:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507881216/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764014.844054:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764014.844058:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764014.844061:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881216:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764014.844068:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881344:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764014.844090:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764014.844103:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764014/real 1789764014] req@ffff8d25d1b09180 x1876699507881344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764014.844115:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507881344/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764014.844124:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764014.844127:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764014.844131:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881344:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764014.844137:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881472:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789764014.844159:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764014.844172:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764014/real 1789764014] req@ffff8d25d1b0ad80 x1876699507881472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764014.844192:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507881472/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764014.844200:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764014.844204:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764014.844207:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881472:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789764014.844214:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507881600:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764014.844237:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764014.844250:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764014/real 1789764014] req@ffff8d25d1b0b800 x1876699507881600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764014.844263:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507881600/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764014.844271:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764014.844275:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764014.844279:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507881600:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764014.844285:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507881728:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764014.844307:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764014.844320:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764014/real 1789764014] req@ffff8d25d1b0b480 x1876699507881728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764014.844332:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507881728/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764039 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764014.844347:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764014.844351:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764014.844354:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507881728:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789764014.907535:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764015.931596:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764016.955557:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764017.979514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764019.003483:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764019.963457:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764019.963465:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764019.963468:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764019.963476:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 3997 00000100:00080000:3.0:1789764019.963483:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764019.963488:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789764019.963514:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08000 x1876699507881856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764019.963535:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764019.963537:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764019.963538:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764019.963541:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3997 00000100:00080000:3.0:1789764019.963543:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764019.963545:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764019.963551:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764019.963558:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0a300 x1876699507881984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764019.963568:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764019.963571:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764019.963573:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764019.963576:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3997 00000100:00080000:3.0:1789764019.963579:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764019.963583:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764019.963588:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764019.963589:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507881856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789764019.963613:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b100 x1876699507882112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764019.963624:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764019.963627:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764019.963629:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764019.963633:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 3997 00000100:00080000:3.0:1789764019.963637:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764019.963639:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764019.963644:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764019.963655:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08380 x1876699507882240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764019.963665:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000800:00000100:0.0:1789764019.963670:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789764019.963671:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764019.963675:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764019.963679:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3997 00000100:00080000:3.0:1789764019.963683:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764019.963686:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764019.963690:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764019.963699:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09500 x1876699507882368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764019.963704:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764019.963707:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764019.963709:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764019.963711:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764019/real 1789764019] req@ffff8d25d1b08000 x1876699507881856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764019.963714:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 3997 00000100:00080000:3.0:1789764019.963718:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764019.963724:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764019.963729:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764019.963742:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08000 x1876699507881856/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764019.963743:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b480 x1876699507882496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764019.963772:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764019.963778:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764019.963783:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507881856:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789764019.963793:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881984:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764019.963814:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764019.963827:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764019/real 1789764019] req@ffff8d25d1b0a300 x1876699507881984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764019.963839:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0a300 x1876699507881984/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764019.963847:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764019.963851:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764019.963855:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0a300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507881984:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764019.963877:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882112:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764019.963896:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764019.963906:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764019/real 1789764019] req@ffff8d25d1b0b100 x1876699507882112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764019.963918:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b100 x1876699507882112/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764019.963925:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764019.963928:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764019.963932:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882112:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764019.963937:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882240:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789764019.963955:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764019.963964:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764019/real 1789764019] req@ffff8d25d1b08380 x1876699507882240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764019.963975:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08380 x1876699507882240/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764019.963983:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764019.963986:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764019.963989:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882240:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789764019.963995:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507882368:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764019.964011:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764019.964020:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764019/real 1789764019] req@ffff8d25d1b09500 x1876699507882368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764019.964031:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09500 x1876699507882368/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764019.964039:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764019.964042:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764019.964045:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09500 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507882368:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764019.964051:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507882496:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764019.964066:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764019.964075:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764019/real 1789764019] req@ffff8d25d1b0b480 x1876699507882496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764019.964086:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b480 x1876699507882496/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764044 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764019.964094:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764019.964097:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764019.964100:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507882496:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789764020.027542:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764021.051573:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764022.075532:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764023.099530:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764024.123529:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:2.0:1789764025.083494:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.083503:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:2.0:1789764025.083507:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764025.083517:0:38:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 4002 00000100:00080000:2.0:1789764025.083525:0:38:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.083531:0:38:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:2.0:1789764025.083564:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023b80 x1876699507882624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764025.083595:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.083598:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764025.083599:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764025.083602:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4002 00000100:00080000:2.0:1789764025.083605:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.083608:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764025.083616:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764025.083628:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022300 x1876699507882752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764025.083636:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.083638:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764025.083640:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764025.083643:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4002 00000100:00080000:2.0:1789764025.083645:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00100000:0.0:1789764025.083649:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507882624:192.168.203.141@tcp:250:kworker.0 00000100:00080000:2.0:1789764025.083650:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764025.083657:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:2.0:1789764025.083670:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020380 x1876699507882880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764025.083681:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.083685:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:2.0:1789764025.083687:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764025.083691:0:38:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4002 00000100:00080000:2.0:1789764025.083696:0:38:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.083700:0:38:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764025.083705:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000800:00000100:0.0:1789764025.083706:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:2.0:1789764025.083717:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8022d80 x1876699507883008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764025.083725:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.083727:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:2.0:1789764025.083728:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:2.0:1789764025.083731:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4002 00000100:00080000:2.0:1789764025.083734:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.083736:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:2.0:1789764025.083739:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764025.083742:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffff8d25c8023b80 x1876699507882624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:2.0:1789764025.083748:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8023100 x1876699507883136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:2.0:1789764025.083766:0:38:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:2.0:1789764025.083778:0:38:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:2.0:1789764025.083780:0:38:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:0.0:1789764025.083786:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023b80 x1876699507882624/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:2.0:1789764025.083787:0:38:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4002 00000100:00080000:2.0:1789764025.083794:0:38:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:2.0:1789764025.083801:0:38:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00080000:0.0:1789764025.083802:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000008:00100000:2.0:1789764025.083808:0:38:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:0.0:1789764025.083809:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764025.083816:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023b80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507882624:192.168.203.141@tcp:250:kworker.0 00000100:00100000:2.0:1789764025.083824:0:38:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25c8020e00 x1876699507883264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764025.083828:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882752:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764025.083856:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764025.083871:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffff8d25c8022300 x1876699507882752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764025.083883:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022300 x1876699507882752/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764025.083892:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764025.083896:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764025.083900:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882752:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764025.083924:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882880:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764025.083940:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764025.083951:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffff8d25c8020380 x1876699507882880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764025.083963:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020380 x1876699507882880/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764025.083978:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764025.083981:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764025.083985:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507882880:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764025.083991:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883008:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789764025.084010:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764025.084021:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffff8d25c8022d80 x1876699507883008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764025.084034:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8022d80 x1876699507883008/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764025.084042:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764025.084046:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764025.084051:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8022d80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883008:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789764025.084059:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507883136:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764025.084076:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764025.084087:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffff8d25c8023100 x1876699507883136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764025.084097:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8023100 x1876699507883136/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764025.084105:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764025.084108:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764025.084112:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8023100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507883136:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764025.084117:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507883264:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764025.084131:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764025.084140:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764025/real 1789764025] req@ffff8d25c8020e00 x1876699507883264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764025.084150:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25c8020e00 x1876699507883264/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764050 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764025.084158:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764025.084161:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764025.084166:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25c8020e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507883264:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789764025.147497:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764026.171575:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764027.195595:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764028.219487:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764029.243506:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:3.0:1789764030.203512:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764030.203521:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:3.0:1789764030.203525:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764030.203718:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 4007 00000100:00080000:3.0:1789764030.203728:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764030.203735:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789764030.203769:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0b800 x1876699507883392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764030.203794:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764030.203798:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764030.203800:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764030.203804:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4007 00000100:00080000:3.0:1789764030.203813:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764030.203818:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764030.203826:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764030.203834:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507883392:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789764030.203841:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0ad80 x1876699507883520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764030.203854:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764030.203857:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764030.203859:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764030.203863:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4007 00000100:00080000:3.0:1789764030.203867:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764030.203871:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764030.203876:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764030.203888:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09180 x1876699507883648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:0.0:1789764030.203891:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789764030.203902:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764030.203905:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:3.0:1789764030.203908:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764030.203912:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4007 00000100:00080000:3.0:1789764030.203916:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764030.203920:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:3.0:1789764030.203925:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:3.0:1789764030.203938:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b0aa00 x1876699507883776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764030.203941:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffff8d25d1b0b800 x1876699507883392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764030.203966:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:0.0:1789764030.203969:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0b800 x1876699507883392/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764030.203972:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:3.0:1789764030.203976:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:3.0:1789764030.203986:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4007 00000100:00080000:0.0:1789764030.203990:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764030.203991:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764030.203997:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00080000:3.0:1789764030.203999:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789764030.204005:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0b800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507883392:192.168.203.141@tcp:250:kworker.0 00000008:00100000:3.0:1789764030.204006:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764030.204031:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b09c00 x1876699507883904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:0.0:1789764030.204032:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883520:192.168.203.141@tcp:38:kworker.0 00000100:00080000:3.0:1789764030.204050:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:3.0:1789764030.204053:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:3.0:1789764030.204055:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:0.0:1789764030.204061:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:3.0:1789764030.204061:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4007 00000100:00080000:3.0:1789764030.204069:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764030.204074:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789764030.204079:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffff8d25d1b0ad80 x1876699507883520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:3.0:1789764030.204081:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764030.204102:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0ad80 x1876699507883520/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764030.204102:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d1b08a80 x1876699507884032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764030.204128:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764030.204132:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764030.204138:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0ad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883520:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764030.204150:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883648:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764030.204170:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764030.204182:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffff8d25d1b09180 x1876699507883648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764030.204200:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09180 x1876699507883648/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764030.204210:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764030.204213:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764030.204217:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883648:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764030.204223:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883776:192.168.203.141@tcp:8:kworker.0 00000800:00000100:0.0:1789764030.204239:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764030.204248:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffff8d25d1b0aa00 x1876699507883776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764030.204266:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b0aa00 x1876699507883776/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764030.204274:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764030.204277:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764030.204280:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b0aa00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507883776:192.168.203.141@tcp:8:kworker.0 00000100:00100000:0.0:1789764030.204303:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507883904:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764030.204319:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764030.204329:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffff8d25d1b09c00 x1876699507883904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764030.204342:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b09c00 x1876699507883904/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764030.204349:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764030.204352:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764030.204356:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b09c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507883904:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764030.204362:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507884032:192.168.203.141@tcp:38:kworker.0 00000800:00000100:0.0:1789764030.204394:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764030.204405:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764030/real 1789764030] req@ffff8d25d1b08a80 x1876699507884032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:0.0:1789764030.204417:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d1b08a80 x1876699507884032/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764055 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764030.204431:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764030.204434:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764030.204438:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d1b08a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507884032:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789764030.267503:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764031.291538:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764032.315493:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764033.339552:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764034.363513:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764035.323511:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764035.323521:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764035.323525:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764035.323534:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 4013 00000100:00080000:0.0:1789764035.323543:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764035.323549:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789764035.323584:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8380 x1876699507884160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764035.323612:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764035.323615:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764035.323617:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764035.323621:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4013 00000100:00080000:0.0:1789764035.323625:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764035.323630:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764035.323639:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764035.323647:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507884160:192.168.203.141@tcp:250:kworker.0 00000100:00100000:0.0:1789764035.323654:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba680 x1876699507884288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764035.323665:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764035.323669:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764035.323671:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764035.323675:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4013 00000100:00080000:0.0:1789764035.323679:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764035.323683:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764035.323688:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789764035.323698:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764035.323702:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9c00 x1876699507884416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764035.323712:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764035.323716:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764035.323717:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00100000:3.0:1789764035.323729:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffff8d25d25b8380 x1876699507884160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764035.323730:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4013 00000100:00080000:0.0:1789764035.323737:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764035.323743:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789764035.323747:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8380 x1876699507884160/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789764035.323750:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00080000:3.0:1789764035.323757:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.323761:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:0.0:1789764035.323769:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8700 x1876699507884544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764035.323772:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8380 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507884160:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789764035.323787:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764035.323790:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764035.323793:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764035.323797:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4013 00000100:00100000:3.0:1789764035.323799:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507884288:192.168.203.141@tcp:38:kworker.0 00000100:00080000:0.0:1789764035.323802:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764035.323809:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764035.323815:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000800:00000100:3.0:1789764035.323819:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:0.0:1789764035.323833:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb100 x1876699507884672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764035.323834:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffff8d25d25ba680 x1876699507884288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764035.323856:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789764035.323861:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba680 x1876699507884288/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764035.323863:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764035.323867:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764035.323876:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4013 00000100:00080000:3.0:1789764035.323877:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:0.0:1789764035.323880:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:3.0:1789764035.323884:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00080000:0.0:1789764035.323890:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789764035.323893:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba680 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507884288:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789764035.323898:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:3.0:1789764035.323902:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507884416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:0.0:1789764035.323912:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8e00 x1876699507884800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764035.323926:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764035.323939:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffff8d25d25b9c00 x1876699507884416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764035.323949:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9c00 x1876699507884416/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764035.323956:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.323958:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764035.323961:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9c00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507884416:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764035.323966:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507884544:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789764035.323983:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764035.323991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffff8d25d25b8700 x1876699507884544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764035.324002:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8700 x1876699507884544/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764035.324009:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.324012:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764035.324015:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8700 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507884544:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789764035.324026:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507884672:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764035.324039:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764035.324046:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffff8d25d25bb100 x1876699507884672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764035.324056:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb100 x1876699507884672/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764035.324061:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.324062:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764035.324064:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb100 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507884672:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764035.324067:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507884800:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764035.324075:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764035.324080:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764035/real 1789764035] req@ffff8d25d25b8e00 x1876699507884800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764035.324086:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8e00 x1876699507884800/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764060 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764035.324090:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764035.324091:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764035.324093:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8e00 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507884800:192.168.203.141@tcp:38:kworker.0 00010000:00010000:0.0:1789764035.387541:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764036.411555:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764037.435554:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764038.459600:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764039.483587:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764040.443481:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764040.443489:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764040.443492:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764040.443499:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 4018 00000100:00080000:0.0:1789764040.443507:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764040.443512:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789764040.443544:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb800 x1876699507884928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764040.443566:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764040.443569:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764040.443570:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764040.443573:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4018 00000100:00080000:0.0:1789764040.443575:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764040.443577:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764040.443584:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764040.443592:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9f80 x1876699507885056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764040.443599:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764040.443601:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764040.443602:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764040.443604:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4018 00000100:00080000:0.0:1789764040.443606:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764040.443609:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764040.443612:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764040.443618:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bb480 x1876699507885184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764040.443625:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507884928:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789764040.443626:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764040.443632:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764040.443635:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764040.443638:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4018 00000100:00080000:0.0:1789764040.443640:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764040.443642:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764040.443645:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764040.443653:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507885312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764040.443663:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764040.443664:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764040.443666:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764040.443668:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4018 00000100:00080000:0.0:1789764040.443671:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764040.443673:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764040.443675:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764040.443686:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507885440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764040.443693:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789764040.443696:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764040.443700:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764040.443701:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764040.443704:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4018 00000100:00080000:0.0:1789764040.443707:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764040.443709:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764040.443712:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764040.443718:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507885568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764040.443736:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764040/real 1789764040] req@ffff8d25d25bb800 x1876699507884928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764040.443754:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb800 x1876699507884928/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764040.443765:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764040.443769:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764040.443773:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb800 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507884928:192.168.203.141@tcp:250:kworker.0 00000100:00100000:3.0:1789764040.443780:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885056:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764040.443800:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764040.443812:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764040/real 1789764040] req@ffff8d25d25b9f80 x1876699507885056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764040.443824:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9f80 x1876699507885056/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764040.443832:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764040.443835:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764040.443838:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9f80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885056:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764040.443858:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885184:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764040.443876:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764040.443887:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764040/real 1789764040] req@ffff8d25d25bb480 x1876699507885184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764040.443898:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bb480 x1876699507885184/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764040.443906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764040.443909:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764040.443912:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bb480 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885184:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764040.443918:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885312:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789764040.443934:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764040.443943:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764040/real 1789764040] req@ffff8d25d25bbb80 x1876699507885312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764040.443954:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507885312/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764040.443962:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764040.443965:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764040.443968:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885312:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789764040.443974:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507885440:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764040.443990:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764040.443999:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764040/real 1789764040] req@ffff8d25d25b9180 x1876699507885440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764040.444010:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507885440/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764040.444016:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764040.444019:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764040.444022:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507885440:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764040.444028:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507885568:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764040.444044:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764040.444054:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764040/real 1789764040] req@ffff8d25d25b8a80 x1876699507885568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764040.444065:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507885568/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764065 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764040.444073:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764040.444075:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764040.444078:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507885568:192.168.203.141@tcp:38:kworker.0 00010000:00010000:3.0:1789764040.507510:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764041.531552:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:2.0:1789764042.555514:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:3.0:1789764043.579566:0:6393:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00010000:00010000:0.0:1789764044.603491:0:12645:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) 00000100:00080000:0.0:1789764045.563459:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7c97d591-104c-4fe1-aa51-5b4983d2eb89->MGS: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764045.563468:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) MGS: starting recovery 00000100:00080000:0.0:1789764045.563472:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764045.563480:0:13229:0:(import.c:510:import_select_connection()) MGC192.168.203.141@tcp: connect to NID 192.168.203.141@tcp last attempt 4023 00000100:00080000:0.0:1789764045.563488:0:13229:0:(import.c:589:import_select_connection()) MGC192.168.203.141@tcp: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764045.563493:0:13229:0:(import.c:624:import_select_connection()) MGC192.168.203.141@tcp: import ffff8d25c5404000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:0.0:1789764045.563528:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bad80 x1876699507885696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764045.563554:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764045.563557:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764045.563559:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764045.563563:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4023 00000100:00080000:0.0:1789764045.563566:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764045.563569:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c5405000: import ffff8d25c8065000 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764045.563579:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764045.563592:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25ba300 x1876699507885824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00100000:3.0:1789764045.563602:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507885696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789764045.563604:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764045.563611:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764045.563613:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764045.563617:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4023 00000100:00080000:0.0:1789764045.563619:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764045.563623:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c5405000: import ffff8d25c8061800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764045.563628:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764045.563640:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8000 x1876699507885952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764045.563653:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) 7fdb621d-3376-4597-a0ab-d89800273221->lustre-OST0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764045.563656:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-OST0000_UUID: starting recovery 00000100:00080000:0.0:1789764045.563658:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from DISCONN to CONNECTING 00000800:00000100:3.0:1789764045.563662:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00080000:0.0:1789764045.563665:0:13229:0:(import.c:510:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: connect to NID 192.168.203.141@tcp last attempt 4023 00000100:00080000:0.0:1789764045.563670:0:13229:0:(import.c:589:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764045.563674:0:13229:0:(import.c:624:import_select_connection()) lustre-OST0000-osc-ffff8d25c5405000: import ffff8d25c8066800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000008:00100000:0.0:1789764045.563679:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xa0467af2e3640038 ocd_version: 34683244 ocd_grant: 1703936, lost: 0. 00000100:00100000:0.0:1789764045.563692:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b8a80 x1876699507886080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:0.0:1789764045.563701:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0000_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00100000:3.0:1789764045.563703:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764045/real 1789764045] req@ffff8d25d25bad80 x1876699507885696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:0.0:1789764045.563705:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0000_UUID: starting recovery 00000100:00080000:0.0:1789764045.563708:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764045.563716:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4023 00000100:00080000:0.0:1789764045.563722:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764045.563726:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0000-mdc-ffff8d25c80cd000: import ffff8d25c810f800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789764045.563730:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bad80 x1876699507885696/t0(0) o250->MGC192.168.203.141@tcp@192.168.203.141@tcp:26/25 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000008:00100000:0.0:1789764045.563732:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00080000:3.0:1789764045.563746:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c5404000 MGS: changing import state from CONNECTING to DISCONN 00000100:00100000:0.0:1789764045.563748:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25b9180 x1876699507886208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000100:00080000:3.0:1789764045.563752:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of MGS on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764045.563759:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bad80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7c97d591-104c-4fe1-aa51-5b4983d2eb89:2415:1876699507885696:192.168.203.141@tcp:250:kworker.0 00000100:00080000:0.0:1789764045.563765:0:13229:0:(pinger.c:229:ptlrpc_pinger_process_import()) e0668622-7dd7-45f4-a020-184f7e88cc06->lustre-MDT0001_UUID: level DISCONN/3 force 0 force_next 0 deactive 0 pingable 1 suppress 0 00000100:00080000:0.0:1789764045.563769:0:13229:0:(pinger.c:237:ptlrpc_pinger_process_import()) lustre-MDT0001_UUID: starting recovery 00000100:00080000:0.0:1789764045.563771:0:13229:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from DISCONN to CONNECTING 00000100:00080000:0.0:1789764045.563775:0:13229:0:(import.c:510:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: connect to NID 192.168.203.141@tcp last attempt 4023 00000100:00080000:0.0:1789764045.563779:0:13229:0:(import.c:589:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: tried all connections, increasing latency to 20s 00000100:00080000:0.0:1789764045.563785:0:13229:0:(import.c:624:import_select_connection()) lustre-MDT0001-mdc-ffff8d25c80cd000: import ffff8d25c810b800 using connection 192.168.203.141@tcp/192.168.203.141@tcp 00000100:00100000:3.0:1789764045.563785:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885824:192.168.203.141@tcp:38:kworker.0 00000008:00100000:0.0:1789764045.563793:0:13229:0:(osc_request.c:3830:osc_reconnect()) ocd_connect_flags: 0xae7e7e7be374d3b8 ocd_version: 34683244 ocd_grant: 2097152, lost: 0. 00000100:00100000:0.0:1789764045.563806:0:13229:0:(import.c:826:ptlrpc_connect_import_locked()) @@@ (re)connect request (timeout 5) req@ffff8d25d25bbb80 x1876699507886336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 0 dl 0 ref 1 fl New:NQU/0/ffffffff rc 0/-1 job:'' uid:4294967295 gid:4294967295 projid:4294967295 00000800:00000100:3.0:1789764045.563815:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764045.563831:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764045/real 1789764045] req@ffff8d25d25ba300 x1876699507885824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764045.563844:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25ba300 x1876699507885824/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764045.563852:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8065000 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764045.563854:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764045.563857:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25ba300 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885824:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764045.563863:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885952:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764045.563879:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764045.563888:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764045/real 1789764045] req@ffff8d25d25b8000 x1876699507885952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764045.563899:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8000 x1876699507885952/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c5405000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764045.563906:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8061800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764045.563910:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764045.563913:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8000 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507885952:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764045.563918:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507886080:192.168.203.141@tcp:8:kworker.0 00000800:00000100:3.0:1789764045.563932:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764045.563940:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764045/real 1789764045] req@ffff8d25d25b8a80 x1876699507886080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764045.563950:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b8a80 x1876699507886080/t0(0) o8->lustre-OST0000-osc-ffff8d25c5405000@192.168.203.141@tcp:28/4 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764045.563958:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c8066800 lustre-OST0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764045.563961:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-OST0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764045.563964:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b8a80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:7fdb621d-3376-4597-a0ab-d89800273221:2415:1876699507886080:192.168.203.141@tcp:8:kworker.0 00000100:00100000:3.0:1789764045.563969:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507886208:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764045.563982:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764045.563991:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764045/real 1789764045] req@ffff8d25d25b9180 x1876699507886208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764045.564001:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25b9180 x1876699507886208/t0(0) o38->lustre-MDT0000-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764045.564008:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810f800 lustre-MDT0000_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764045.564011:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0000_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764045.564013:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25b9180 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507886208:192.168.203.141@tcp:38:kworker.0 00000100:00100000:3.0:1789764045.564027:0:2415:0:(client.c:1919:ptlrpc_send_new_req()) Sending RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507886336:192.168.203.141@tcp:38:kworker.0 00000800:00000100:3.0:1789764045.564040:0:2415:0:(socklnd_cb.c:906:ksocknal_launch_packet()) No usable routes to 12345-192.168.203.141@tcp 00000100:00100000:3.0:1789764045.564049:0:2415:0:(client.c:2503:ptlrpc_expire_one_request()) @@@ Request sent has failed due to network error: [sent 1789764045/real 1789764045] req@ffff8d25d25bbb80 x1876699507886336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQr/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00100000:3.0:1789764045.564059:0:2415:0:(client.c:2534:ptlrpc_expire_one_request()) @@@ err -110, sent_state=CONNECTING (now=CONNECTING) req@ffff8d25d25bbb80 x1876699507886336/t0(0) o38->lustre-MDT0001-mdc-ffff8d25c80cd000@192.168.203.141@tcp:12/10 lens 520/544 e 0 to 1 dl 1789764070 ref 1 fl Rpc:eXNQU/200/ffffffff rc 0/-1 job:'kworker.0' uid:0 gid:0 projid:4294967295 00000100:00080000:3.0:1789764045.564066:0:2415:0:(import.c:65:import_set_state_nolock()) ffff8d25c810b800 lustre-MDT0001_UUID: changing import state from CONNECTING to DISCONN 00000100:00080000:3.0:1789764045.564069:0:2415:0:(import.c:1511:ptlrpc_connect_interpret()) recovery of lustre-MDT0001_UUID on 192.168.203.141@tcp failed (-110) 00000100:00100000:3.0:1789764045.564072:0:2415:0:(client.c:2414:ptlrpc_check_set()) Completed RPC req@ffff8d25d25bbb80 pname:cluuid:pid:xid:nid:opc:job ptlrpcd_rcv:e0668622-7dd7-45f4-a020-184f7e88cc06:2415:1876699507886336:192.168.203.141@tcp:38:kworker.0 00010000:00010000:2.0:1789764045.627546:0:6394:0:(ldlm_lockd.c:2804:ldlm_bl_get_work()) No blwi found in queue (no bl locks in queue) Debug log: 86572 lines, 86572 kept, 0 dropped, 0 bad.